diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index d4ed5970..37aa49b3 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -4,7 +4,7 @@ inputs: go-version: description: "Go version to install" required: true - default: "1.19.x" + default: "1.21.x" use-go-cache: description: "Restore go cache" required: true diff --git a/.golangci.yaml b/.golangci.yaml index dfd5fe5c..39a9e2e3 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,15 +1,17 @@ -run: - timeout: 10m -linters-settings: - funlen: - lines: 75 +issues: + max-same-issues: 25 + + # TODO: enable this when we have coverage on docstring comments +# # The list of ids of default excludes to include or disable. +# include: +# - EXC0002 # disable excluding of issues about comments from golint + linters: # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint disable-all: true enable: - asciicheck - bodyclose - - depguard - dogsled - dupl - errcheck @@ -37,18 +39,38 @@ linters: - unused - whitespace +linters-settings: + funlen: + # Checks the number of lines in a function. + # If lower than 0, disable the check. + # Default: 60 + lines: 70 + # Checks the number of statements in a function. + # If lower than 0, disable the check. + # Default: 40 + statements: 50 +output: + uniq-by-line: false +run: + timeout: 10m + # do not enable... -# - deadcode # The owner seems to have abandoned the linter. Replaced by "unused". +# - deadcode # The owner seems to have abandoned the linter. Replaced by "unused". +# - depguard # We don't have a configuration for this yet +# - goprintffuncname # does not catch all cases and there are exceptions +# - nakedret # does not catch all cases and should not fail a build # - gochecknoglobals # - gochecknoinits # this is too aggressive +# - rowserrcheck disabled per generics https://github.com/golangci/golangci-lint/issues/2649 # - godot # - godox # - goerr113 -# - golint # deprecated -# - gomnd # this is too aggressive -# - interfacer # this is a good idea, but is no longer supported and is prone to false positives -# - lll # without a way to specify per-line exception cases, this is not usable -# - maligned # this is an excellent linter, but tricky to optimize and we are not sensitive to memory layout optimizations +# - goimports # we're using gosimports now instead to account for extra whitespaces (see https://github.com/golang/go/issues/20818) +# - golint # deprecated +# - gomnd # this is too aggressive +# - interfacer # this is a good idea, but is no longer supported and is prone to false positives +# - lll # without a way to specify per-line exception cases, this is not usable +# - maligned # this is an excellent linter, but tricky to optimize and we are not sensitive to memory layout optimizations # - nestif # - nolintlint # as of go1.19 this conflicts with the behavior of gofmt, which is a deal-breaker (lint-fix will still fail when running lint) # - prealloc # following this rule isn't consistently a good idea, as it sometimes forces unnecessary allocations that result in less idiomatic code diff --git a/Makefile b/Makefile index 38651e08..dced9302 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ LINT_CMD = $(TEMP_DIR)/golangci-lint run --tests=false --config .golangci.yaml GOIMPORTS_CMD := $(TEMP_DIR)/gosimports -local github.com/anchore # Tool versions ################################# -GOLANGCILINT_VERSION := v1.52.2 +GOLANGCILINT_VERSION := v1.56.0 GOSIMPORTS_VERSION := v0.3.8 BOUNCER_VERSION := v0.4.0 CHRONICLE_VERSION := v0.6.0 diff --git a/client.go b/client.go index b5056898..4ff3286c 100644 --- a/client.go +++ b/client.go @@ -2,23 +2,18 @@ package stereoscope import ( "context" + "errors" "fmt" - "runtime" + "strings" "github.com/wagoodman/go-partybus" + "github.com/anchore/go-collections" "github.com/anchore/go-logger" "github.com/anchore/stereoscope/internal/bus" - containerdClient "github.com/anchore/stereoscope/internal/containerd" - dockerClient "github.com/anchore/stereoscope/internal/docker" "github.com/anchore/stereoscope/internal/log" - "github.com/anchore/stereoscope/internal/podman" "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" - "github.com/anchore/stereoscope/pkg/image/containerd" - "github.com/anchore/stereoscope/pkg/image/docker" - "github.com/anchore/stereoscope/pkg/image/oci" - "github.com/anchore/stereoscope/pkg/image/sif" ) var rootTempDirGenerator = file.NewTempDirGenerator("stereoscope") @@ -69,151 +64,59 @@ func WithPlatform(platform string) Option { } } +// GetImage parses the user provided image string and provides an image object; +// note: the source where the image should be referenced from is automatically inferred. +func GetImage(ctx context.Context, imgStr string, options ...Option) (*image.Image, error) { + // look for a known source scheme like docker: + source, imgStr := ExtractSchemeSource(imgStr, allProviderTags()...) + return getImageFromSource(ctx, imgStr, source, options...) +} + // GetImageFromSource returns an image from the explicitly provided source. func GetImageFromSource(ctx context.Context, imgStr string, source image.Source, options ...Option) (*image.Image, error) { - log.Debugf("image: source=%+v location=%+v", source, imgStr) - - var cfg config - for _, option := range options { - if option == nil { - continue - } - if err := option(&cfg); err != nil { - return nil, fmt.Errorf("unable to parse option: %w", err) - } - } - - provider, cleanup, err := selectImageProvider(imgStr, source, cfg) - if cleanup != nil { - defer cleanup() - } - if err != nil { - return nil, err - } - - img, err := provider.Provide(ctx, cfg.AdditionalMetadata...) - if err != nil { - return nil, fmt.Errorf("unable to use %s source: %w", source, err) - } - - err = img.Read() - if err != nil { - return nil, fmt.Errorf("could not read image: %+v", err) + if source == "" { + return nil, fmt.Errorf("source not provided, please specify a valid source tag") } - - return img, nil + return getImageFromSource(ctx, imgStr, source, options...) } -// nolint:funlen -func selectImageProvider(imgStr string, source image.Source, cfg config) (image.Provider, func(), error) { - var provider image.Provider - tempDirGenerator := rootTempDirGenerator.NewGenerator() - platformSelectionUnsupported := fmt.Errorf("specified platform=%q however image source=%q does not support selecting platform", cfg.Platform.String(), source.String()) - - cleanup := func() {} - - switch source { - case image.DockerTarballSource: - if cfg.Platform != nil { - return nil, cleanup, platformSelectionUnsupported - } - // note: the imgStr is the path on disk to the tar file - provider = docker.NewProviderFromTarball(imgStr, tempDirGenerator) - case image.ContainerdDaemonSource: - c, err := containerdClient.GetClient() - if err != nil { - return nil, cleanup, err - } - - cleanup = func() { - if err := c.Close(); err != nil { - log.Errorf("unable to close docker client: %+v", err) - } - } - - provider, err = containerd.NewProviderFromDaemon(imgStr, tempDirGenerator, c, containerdClient.Namespace(), cfg.Registry, cfg.Platform) - if err != nil { - return nil, cleanup, err - } - case image.DockerDaemonSource: - c, err := dockerClient.GetClient() - if err != nil { - return nil, cleanup, err - } - - cleanup = func() { - if err := c.Close(); err != nil { - log.Errorf("unable to close docker client: %+v", err) - } - } +func getImageFromSource(ctx context.Context, imgStr string, source image.Source, options ...Option) (*image.Image, error) { + log.Debugf("image: source=%+v location=%+v", source, imgStr) - provider, err = docker.NewProviderFromDaemon(imgStr, tempDirGenerator, c, cfg.Platform) - if err != nil { - return nil, cleanup, err - } - case image.PodmanDaemonSource: - c, err := podman.GetClient() - if err != nil { - return nil, cleanup, err - } + // apply ImageProviderConfig config + cfg := config{} + if err := applyOptions(&cfg, options...); err != nil { + return nil, err + } - cleanup = func() { - if err := c.Close(); err != nil { - log.Errorf("unable to close docker client: %+v", err) - } + // select image provider + providers := collections.TaggedValueSet[image.Provider]{}.Join( + ImageProviders(ImageProviderConfig{ + UserInput: imgStr, + Platform: cfg.Platform, + Registry: cfg.Registry, + })..., + ) + if source != "" { + source = strings.ToLower(strings.TrimSpace(source)) + providers = providers.Select(source) + if len(providers) == 0 { + return nil, fmt.Errorf("unable to find image providers matching: '%s'", source) } + } - provider, err = docker.NewProviderFromDaemon(imgStr, tempDirGenerator, c, cfg.Platform) + var errs []error + for _, provider := range providers.Values() { + img, err := provider.Provide(ctx) if err != nil { - return nil, cleanup, err + errs = append(errs, err) } - case image.OciDirectorySource: - if cfg.Platform != nil { - return nil, cleanup, platformSelectionUnsupported + if img != nil { + err = applyAdditionalMetadata(img, cfg.AdditionalMetadata...) + return img, err } - provider = oci.NewProviderFromPath(imgStr, tempDirGenerator) - case image.OciTarballSource: - if cfg.Platform != nil { - return nil, cleanup, platformSelectionUnsupported - } - provider = oci.NewProviderFromTarball(imgStr, tempDirGenerator) - case image.OciRegistrySource: - defaultPlatformIfNil(&cfg) - provider = oci.NewProviderFromRegistry(imgStr, tempDirGenerator, cfg.Registry, cfg.Platform) - case image.SingularitySource: - if cfg.Platform != nil { - return nil, cleanup, platformSelectionUnsupported - } - provider = sif.NewProviderFromPath(imgStr, tempDirGenerator) - default: - return nil, cleanup, fmt.Errorf("unable to determine image source") - } - return provider, cleanup, nil -} - -// defaultPlatformIfNil sets the platform to use the host's architecture -// if no platform was specified. The OCI registry provider uses "linux/amd64" -// as a hard-coded default platform, which has surprised customers -// running stereoscope on non-amd64 hosts. If platform is already -// set on the config, or the code can't generate a matching platform, -// do nothing. -func defaultPlatformIfNil(cfg *config) { - if cfg.Platform == nil { - p, err := image.NewPlatform(fmt.Sprintf("linux/%s", runtime.GOARCH)) - if err == nil { - cfg.Platform = p - } - } -} - -// GetImage parses the user provided image string and provides an image object; -// note: the source where the image should be referenced from is automatically inferred. -func GetImage(ctx context.Context, userStr string, options ...Option) (*image.Image, error) { - source, imgStr, err := image.DetectSource(userStr) - if err != nil { - return nil, err } - return GetImageFromSource(ctx, imgStr, source, options...) + return nil, fmt.Errorf("unable to detect input for '%s', errs: %w", imgStr, errors.Join(errs...)) } func SetLogger(logger logger.Logger) { diff --git a/config.go b/config.go deleted file mode 100644 index d8c163da..00000000 --- a/config.go +++ /dev/null @@ -1,11 +0,0 @@ -package stereoscope - -import ( - "github.com/anchore/stereoscope/pkg/image" -) - -type config struct { - Registry image.RegistryOptions - AdditionalMetadata []image.AdditionalMetadata - Platform *image.Platform -} diff --git a/deprecated.go b/deprecated.go new file mode 100644 index 00000000..3d4d20d9 --- /dev/null +++ b/deprecated.go @@ -0,0 +1,28 @@ +package stereoscope + +import ( + "slices" + "strings" +) + +// ExtractSchemeSource parses a string with any colon-delimited prefix and validates it against the set +// of known provider tags, returning a valid source name and input string to use for GetImageFromSource +// +// NOTE: since it is now possible to select which providers to use, using schemes +// in the user input text is not necessary and should be avoided due to some ambiguity this introduces +func ExtractSchemeSource(userInput string, sources ...string) (source, newInput string) { + const SchemeSeparator = ":" + parts := strings.SplitN(userInput, SchemeSeparator, 2) + if len(parts) < 2 { + return "", userInput + } + // the user may have provided a source hint (or this is a split from a path or docker image reference, we aren't certain yet) + sourceHint := parts[0] + sourceHint = strings.TrimSpace(strings.ToLower(sourceHint)) + // check the hint against the possible tags + if slices.Contains(sources, sourceHint) { + return sourceHint, parts[1] + } + // did not have any matching tags, scheme is not a valid provider scheme + return "", userInput +} diff --git a/go.mod b/go.mod index 3c0e3201..650b7888 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/anchore/stereoscope -go 1.19 +go 1.21.0 require ( github.com/GoogleCloudPlatform/docker-credential-gcr v2.0.5+incompatible @@ -111,6 +111,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) +require github.com/anchore/go-collections v0.0.0-20240216171411-9321230ce537 + require ( github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/containerd/log v0.1.0 // indirect diff --git a/go.sum b/go.sum index 6f0cb93a..035e3642 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= +github.com/anchore/go-collections v0.0.0-20240216171411-9321230ce537 h1:GjNGuwK5jWjJMyVppBjYS54eOiiSNv4Ba869k4wh72Q= +github.com/anchore/go-collections v0.0.0-20240216171411-9321230ce537/go.mod h1:1aiktV46ATCkuVg0O573ZrH56BUawTECPETbZyBcqT8= github.com/anchore/go-logger v0.0.0-20220728155337-03b66a5207d8 h1:imgMA0gN0TZx7PSa/pdWqXadBvrz8WsN6zySzCe4XX0= github.com/anchore/go-logger v0.0.0-20220728155337-03b66a5207d8/go.mod h1:+gPap4jha079qzRTUaehv+UZ6sSdaNwkH0D3b6zhTuk= github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8= @@ -116,6 +118,7 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -190,7 +193,9 @@ github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWK github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= @@ -214,12 +219,14 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e h1:7q6NSFZDeGfvvtIRwBrU/aegEYJYmvev0cHAwo17zZQ= github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs= github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= +github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -338,6 +345,7 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -390,6 +398,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -400,5 +409,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/option.go b/option.go index 15a4792b..854a7304 100644 --- a/option.go +++ b/option.go @@ -1,3 +1,37 @@ package stereoscope +import ( + "errors" + "fmt" + + "github.com/anchore/stereoscope/pkg/image" +) + type Option func(*config) error + +type config struct { + Registry image.RegistryOptions + AdditionalMetadata []image.AdditionalMetadata + Platform *image.Platform +} + +func applyOptions(cfg *config, options ...Option) error { + for _, option := range options { + if option == nil { + continue + } + if err := option(cfg); err != nil { + return fmt.Errorf("unable to parse option: %w", err) + } + } + return nil +} + +func applyAdditionalMetadata(img *image.Image, metadata ...image.AdditionalMetadata) error { + var errs error + for _, userMetadata := range metadata { + err := userMetadata(img) + errs = errors.Join(errs, err) + } + return errs +} diff --git a/pkg/image/containerd/daemon_provider.go b/pkg/image/containerd/daemon_provider.go index d0625f55..da179d89 100644 --- a/pkg/image/containerd/daemon_provider.go +++ b/pkg/image/containerd/daemon_provider.go @@ -24,6 +24,7 @@ import ( "github.com/wagoodman/go-progress" "github.com/anchore/stereoscope/internal/bus" + containerdClient "github.com/anchore/stereoscope/internal/containerd" "github.com/anchore/stereoscope/internal/log" "github.com/anchore/stereoscope/pkg/event" "github.com/anchore/stereoscope/pkg/file" @@ -31,72 +32,75 @@ import ( stereoscopeDocker "github.com/anchore/stereoscope/pkg/image/docker" ) +const Daemon image.Source = image.ContainerdDaemonSource + +// NewDaemonProvider creates a new provider instance for a specific image that will later be cached to the given directory. +func NewDaemonProvider(tmpDirGen *file.TempDirGenerator, registryOptions image.RegistryOptions, namespace string, imageStr string, platform *image.Platform) image.Provider { + if namespace == "" { + namespace = namespaces.Default + } + + return &daemonImageProvider{ + imageStr: imageStr, + tmpDirGen: tmpDirGen, + platform: platform, + namespace: namespace, + registryOptions: registryOptions, + } +} + var mb = math.Pow(2, 20) -// DaemonImageProvider is a image.Provider capable of fetching and representing a docker image from the containerd daemon API. -type DaemonImageProvider struct { +// daemonImageProvider is an image.Provider capable of fetching and representing a docker image from the containerd daemon API +type daemonImageProvider struct { imageStr string tmpDirGen *file.TempDirGenerator - client *containerd.Client platform *image.Platform namespace string registryOptions image.RegistryOptions } +func (p *daemonImageProvider) Name() string { + return Daemon +} + type daemonProvideProgress struct { EstimateProgress *progress.TimedProgress ExportProgress *progress.Manual Stage *progress.Stage } -// NewProviderFromDaemon creates a new provider instance for a specific image that will later be cached to the given directory. -func NewProviderFromDaemon(imgStr string, tmpDirGen *file.TempDirGenerator, c *containerd.Client, namespace string, registryOptions image.RegistryOptions, platform *image.Platform) (*DaemonImageProvider, error) { - ref, err := name.ParseReference(imgStr, name.WithDefaultRegistry("")) +func (p *daemonImageProvider) Provide(ctx context.Context) (*image.Image, error) { + client, err := containerdClient.GetClient() if err != nil { - return nil, err - } - tag, ok := ref.(name.Tag) - if ok { - imgStr = tag.Name() + return nil, fmt.Errorf("containerd not available: %w", err) } - if namespace == "" { - namespace = namespaces.Default - } - - return &DaemonImageProvider{ - imageStr: imgStr, - tmpDirGen: tmpDirGen, - client: c, - platform: platform, - namespace: namespace, - registryOptions: registryOptions, - }, nil -} + defer func() { + if err := client.Close(); err != nil { + log.Errorf("unable to close containerd client: %+v", err) + } + }() -// Provide an image object that represents the cached docker image tar fetched from a containerd daemon. -func (p *DaemonImageProvider) Provide(ctx context.Context, userMetadata ...image.AdditionalMetadata) (*image.Image, error) { ctx = namespaces.WithNamespace(ctx, p.namespace) - resolvedImage, resolvedPlatform, err := p.pullImageIfMissing(ctx) + resolvedImage, resolvedPlatform, err := p.pullImageIfMissing(ctx, client) if err != nil { return nil, err } - tarFileName, err := p.saveImage(ctx, resolvedImage) + tarFileName, err := p.saveImage(ctx, client, resolvedImage) if err != nil { return nil, err } // use the existing tarball provider to process what was pulled from the containerd daemon - return stereoscopeDocker.NewProviderFromTarball(tarFileName, p.tmpDirGen). - Provide(ctx, - withMetadata(resolvedPlatform, userMetadata, p.imageStr)..., - ) + return stereoscopeDocker.NewArchiveProvider(p.tmpDirGen, tarFileName, withMetadata(resolvedPlatform, p.imageStr)...). + Provide(ctx) } // pull a containerd image -func (p *DaemonImageProvider) pull(ctx context.Context, resolvedImage string) (containerd.Image, error) { +func (p *daemonImageProvider) pull(ctx context.Context, client *containerd.Client, resolvedImage string) (containerd.Image, error) { var platformStr string if p.platform != nil { platformStr = p.platform.String() @@ -112,7 +116,7 @@ func (p *DaemonImageProvider) pull(ctx context.Context, resolvedImage string) (c bus.Publish(partybus.Event{ Type: event.PullContainerdImage, Source: resolvedImage, - Value: newPullStatus(p.client, ongoing).start(ctx), + Value: newPullStatus(client, ongoing).start(ctx), }) h := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { @@ -133,12 +137,9 @@ func (p *DaemonImageProvider) pull(ctx context.Context, resolvedImage string) (c return nil, fmt.Errorf("unable to prepare pull options: %w", err) } options = append(options, containerd.WithImageHandler(h)) - if platformStr != "" { - options = append(options, containerd.WithPlatform(platformStr)) - } // note: this will return an image object with the platform correctly set (if it exists) - resp, err := p.client.Pull(ctx, resolvedImage, options...) + resp, err := client.Pull(ctx, resolvedImage, options...) if err != nil { return nil, fmt.Errorf("pull failed: %w", err) } @@ -146,7 +147,7 @@ func (p *DaemonImageProvider) pull(ctx context.Context, resolvedImage string) (c return resp, nil } -func (p *DaemonImageProvider) pullOptions(ctx context.Context, ref name.Reference) ([]containerd.RemoteOpt, error) { +func (p *daemonImageProvider) pullOptions(ctx context.Context, ref name.Reference) ([]containerd.RemoteOpt, error) { var options = []containerd.RemoteOpt{ containerd.WithPlatform(p.platform.String()), } @@ -204,12 +205,12 @@ func (p *DaemonImageProvider) pullOptions(ctx context.Context, ref name.Referenc return options, nil } -func (p *DaemonImageProvider) resolveImage(ctx context.Context, imageStr string) (string, *platforms.Platform, error) { +func (p *daemonImageProvider) resolveImage(ctx context.Context, client *containerd.Client, imageStr string) (string, *platforms.Platform, error) { // check if the image exists locally // note: you can NEVER depend on the GetImage() call to return an object with a platform set (even if you specify // a reference to a specific manifest via digest... not a digest for a manifest list!). - img, err := p.client.GetImage(ctx, imageStr) + img, err := client.GetImage(ctx, imageStr) if err != nil { // no image found return imageStr, nil, err @@ -221,12 +222,12 @@ func (p *DaemonImageProvider) resolveImage(ctx context.Context, imageStr string) } processManifest := func(imageStr string, manifestDesc ocispec.Descriptor) (string, *platforms.Platform, error) { - manifest, err := p.fetchManifest(ctx, manifestDesc) + manifest, err := p.fetchManifest(ctx, client, manifestDesc) if err != nil { return "", nil, err } - platform, err := p.fetchPlatformFromConfig(ctx, manifest.Config) + platform, err := p.fetchPlatformFromConfig(ctx, client, manifest.Config) if err != nil { return "", nil, err } @@ -244,7 +245,7 @@ func (p *DaemonImageProvider) resolveImage(ctx context.Context, imageStr string) img = nil // let's find the digest for the manifest for the specific architecture we want - by, err := content.ReadBlob(ctx, p.client.ContentStore(), desc) + by, err := content.ReadBlob(ctx, client.ContentStore(), desc) if err != nil { return "", nil, fmt.Errorf("unable to fetch manifest list: %w", err) } @@ -275,7 +276,7 @@ func (p *DaemonImageProvider) resolveImage(ctx context.Context, imageStr string) return "", nil, fmt.Errorf("unexpected mediaType for image: %q", desc.MediaType) } -func (p *DaemonImageProvider) fetchManifest(ctx context.Context, desc ocispec.Descriptor) (*ocispec.Manifest, error) { +func (p *daemonImageProvider) fetchManifest(ctx context.Context, client *containerd.Client, desc ocispec.Descriptor) (*ocispec.Manifest, error) { switch desc.MediaType { case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: // pass @@ -283,7 +284,7 @@ func (p *DaemonImageProvider) fetchManifest(ctx context.Context, desc ocispec.De return nil, fmt.Errorf("unexpected mediaType for image manifest: %q", desc.MediaType) } - by, err := content.ReadBlob(ctx, p.client.ContentStore(), desc) + by, err := content.ReadBlob(ctx, client.ContentStore(), desc) if err != nil { return nil, fmt.Errorf("unable to fetch image manifest: %w", err) } @@ -296,7 +297,7 @@ func (p *DaemonImageProvider) fetchManifest(ctx context.Context, desc ocispec.De return &manifest, nil } -func (p *DaemonImageProvider) fetchPlatformFromConfig(ctx context.Context, desc ocispec.Descriptor) (*platforms.Platform, error) { +func (p *daemonImageProvider) fetchPlatformFromConfig(ctx context.Context, client *containerd.Client, desc ocispec.Descriptor) (*platforms.Platform, error) { switch desc.MediaType { case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig: // pass @@ -304,7 +305,7 @@ func (p *DaemonImageProvider) fetchPlatformFromConfig(ctx context.Context, desc return nil, fmt.Errorf("unexpected mediaType for image config: %q", desc.MediaType) } - by, err := content.ReadBlob(ctx, p.client.ContentStore(), desc) + by, err := content.ReadBlob(ctx, client.ContentStore(), desc) if err != nil { return nil, fmt.Errorf("unable to fetch image config: %w", err) } @@ -317,11 +318,11 @@ func (p *DaemonImageProvider) fetchPlatformFromConfig(ctx context.Context, desc return &cfg, nil } -func (p *DaemonImageProvider) pullImageIfMissing(ctx context.Context) (string, *platforms.Platform, error) { +func (p *daemonImageProvider) pullImageIfMissing(ctx context.Context, client *containerd.Client) (string, *platforms.Platform, error) { p.imageStr = checkRegistryHostMissing(p.imageStr) // try to get the image first before pulling - resolvedImage, resolvedPlatform, err := p.resolveImage(ctx, p.imageStr) + resolvedImage, resolvedPlatform, err := p.resolveImage(ctx, client, p.imageStr) imageStr := resolvedImage if imageStr == "" { @@ -329,12 +330,12 @@ func (p *DaemonImageProvider) pullImageIfMissing(ctx context.Context) (string, * } if err != nil { - _, err := p.pull(ctx, imageStr) + _, err := p.pull(ctx, client, imageStr) if err != nil { return "", nil, err } - resolvedImage, resolvedPlatform, err = p.resolveImage(ctx, imageStr) + resolvedImage, resolvedPlatform, err = p.resolveImage(ctx, client, imageStr) if err != nil { return "", nil, fmt.Errorf("unable to resolve image after pull: %w", err) } @@ -347,7 +348,7 @@ func (p *DaemonImageProvider) pullImageIfMissing(ctx context.Context) (string, * return resolvedImage, resolvedPlatform, nil } -func (p *DaemonImageProvider) validatePlatform(platform *platforms.Platform) error { +func (p *daemonImageProvider) validatePlatform(platform *platforms.Platform) error { if p.platform == nil { return nil } @@ -372,7 +373,7 @@ func (p *DaemonImageProvider) validatePlatform(platform *platforms.Platform) err } // save the image from the containerd daemon to a tar file -func (p *DaemonImageProvider) saveImage(ctx context.Context, resolvedImage string) (string, error) { +func (p *daemonImageProvider) saveImage(ctx context.Context, client *containerd.Client, resolvedImage string) (string, error) { imageTempDir, err := p.tmpDirGen.NewDirectory("containerd-daemon-image") if err != nil { return "", err @@ -390,12 +391,12 @@ func (p *DaemonImageProvider) saveImage(ctx context.Context, resolvedImage strin } }() - is := p.client.ImageService() + is := client.ImageService() exportOpts := []archive.ExportOpt{ archive.WithImage(is, resolvedImage), } - img, err := p.client.GetImage(ctx, resolvedImage) + img, err := client.GetImage(ctx, resolvedImage) if err != nil { return "", fmt.Errorf("unable to fetch image from containerd: %w", err) } @@ -424,7 +425,7 @@ func (p *DaemonImageProvider) saveImage(ctx context.Context, resolvedImage strin providerProgress.Stage.Current = "requesting image from containerd" // containerd export (save) does not return till fully complete - err = p.client.Export(ctx, tempTarFile, exportOpts...) + err = client.Export(ctx, tempTarFile, exportOpts...) if err != nil { return "", fmt.Errorf("unable to save image tar for image=%q: %w", img.Name(), err) } @@ -449,7 +450,7 @@ func exportPlatformComparer(platform *image.Platform) (platforms.MatchComparer, return platforms.OnlyStrict(platformObj), nil } -func (p *DaemonImageProvider) trackSaveProgress(size int64) *daemonProvideProgress { +func (p *daemonImageProvider) trackSaveProgress(size int64) *daemonProvideProgress { // docker image save clocks in at ~40MB/sec on my laptop... mileage may vary, of course :shrug: sec := float64(size) / (mb * 40) approxSaveTime := time.Duration(sec*1000) * time.Millisecond @@ -488,7 +489,7 @@ func prepareReferenceOptions(registryOptions image.RegistryOptions) []name.Optio return options } -func withMetadata(platform *platforms.Platform, userMetadata []image.AdditionalMetadata, ref string) (metadata []image.AdditionalMetadata) { +func withMetadata(platform *platforms.Platform, ref string) (metadata []image.AdditionalMetadata) { if platform != nil { metadata = append(metadata, image.WithArchitecture(platform.Architecture, platform.Variant), @@ -500,9 +501,6 @@ func withMetadata(platform *platforms.Platform, userMetadata []image.AdditionalM // remove digest from ref metadata = append(metadata, image.WithTags(strings.Split(ref, "@")[0])) } - - // apply user-supplied metadata last to override any default behavior - metadata = append(metadata, userMetadata...) return metadata } diff --git a/pkg/image/containerd/daemon_provider_test.go b/pkg/image/containerd/daemon_provider_test.go index 5805b944..b45e0b8b 100644 --- a/pkg/image/containerd/daemon_provider_test.go +++ b/pkg/image/containerd/daemon_provider_test.go @@ -11,53 +11,6 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -func TestNewProviderFromDaemon_imageParsing(t *testing.T) { - tests := []struct { - image string - want string - wantErr require.ErrorAssertionFunc - }{ - { - image: "alpine:sometag", - want: "alpine:sometag", - }, - { - image: "alpine:latest", - want: "alpine:latest", - }, - { - image: "alpine", - want: "alpine:latest", - }, - { - image: "registry.place.io/thing:version", - want: "registry.place.io/thing:version", - }, - { - image: "alpine@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", - want: "alpine@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", - }, - { - image: "alpine:sometag@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", - want: "alpine:sometag@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", - }, - } - for _, tt := range tests { - t.Run(tt.image, func(t *testing.T) { - if tt.wantErr == nil { - tt.wantErr = require.NoError - } - got, err := NewProviderFromDaemon(tt.image, nil, nil, "", image.RegistryOptions{}, nil) - tt.wantErr(t, err) - if err != nil { - return - } - require.NotNil(t, got) - assert.Equal(t, tt.want, got.imageStr) - }) - } -} - func Test_checkRegistryHostMissing(t *testing.T) { tests := []struct { image string diff --git a/pkg/image/docker/daemon_provider.go b/pkg/image/docker/daemon_provider.go index 7954428c..d53229d4 100644 --- a/pkg/image/docker/daemon_provider.go +++ b/pkg/image/docker/daemon_provider.go @@ -23,40 +23,46 @@ import ( "github.com/wagoodman/go-progress" "github.com/anchore/stereoscope/internal/bus" + "github.com/anchore/stereoscope/internal/docker" "github.com/anchore/stereoscope/internal/log" "github.com/anchore/stereoscope/pkg/event" "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" ) -// DaemonImageProvider is a image.Provider capable of fetching and representing a docker image from the docker daemon API. -type DaemonImageProvider struct { - imageStr string - originalImageRef string - tmpDirGen *file.TempDirGenerator - client client.APIClient - platform *image.Platform +const Daemon image.Source = image.DockerDaemonSource + +// NewDaemonProvider creates a new provider instance for a specific image that will later be cached to the given directory +func NewDaemonProvider(tmpDirGen *file.TempDirGenerator, imageStr string, platform *image.Platform) image.Provider { + return NewAPIClientProvider(Daemon, tmpDirGen, imageStr, platform, func() (client.APIClient, error) { + return docker.GetClient() + }) } -// NewProviderFromDaemon creates a new provider instance for a specific image that will later be cached to the given directory. -func NewProviderFromDaemon(imgStr string, tmpDirGen *file.TempDirGenerator, c client.APIClient, platform *image.Platform) (*DaemonImageProvider, error) { - var originalRef string - ref, err := name.ParseReference(imgStr, name.WithDefaultRegistry("")) - if err != nil { - return nil, err +// NewAPIClientProvider creates a new provider for the provided Docker client.APIClient +func NewAPIClientProvider(name string, tmpDirGen *file.TempDirGenerator, imageStr string, platform *image.Platform, newClient apiClientCreator) image.Provider { + return &daemonImageProvider{ + name: name, + tmpDirGen: tmpDirGen, + newAPIClient: newClient, + imageStr: imageStr, + platform: platform, } - tag, ok := ref.(name.Tag) - if ok { - imgStr = tag.Name() - originalRef = tag.String() // blindly takes the original input passed into Tag - } - return &DaemonImageProvider{ - imageStr: imgStr, - originalImageRef: originalRef, - tmpDirGen: tmpDirGen, - client: c, - platform: platform, - }, nil +} + +type apiClientCreator func() (client.APIClient, error) + +// daemonImageProvider is an image.Provider capable of fetching and representing a docker image from the docker daemon API +type daemonImageProvider struct { + name string + tmpDirGen *file.TempDirGenerator + newAPIClient apiClientCreator + imageStr string + platform *image.Platform +} + +func (p *daemonImageProvider) Name() string { + return p.name } type daemonProvideProgress struct { @@ -66,9 +72,9 @@ type daemonProvideProgress struct { } //nolint:staticcheck -func (p *DaemonImageProvider) trackSaveProgress() (*daemonProvideProgress, error) { +func (p *daemonImageProvider) trackSaveProgress(ctx context.Context, apiClient client.APIClient, imageRef string) (*daemonProvideProgress, error) { // fetch the expected image size to estimate and measure progress - inspect, _, err := p.client.ImageInspectWithRaw(context.Background(), p.imageStr) + inspect, _, err := apiClient.ImageInspectWithRaw(ctx, imageRef) if err != nil { return nil, fmt.Errorf("unable to inspect image: %w", err) } @@ -91,7 +97,7 @@ func (p *DaemonImageProvider) trackSaveProgress() (*daemonProvideProgress, error bus.Publish(partybus.Event{ Type: event.FetchImage, - Source: p.imageStr, + Source: imageRef, Value: progress.StagedProgressable(&struct { progress.Stager *progress.Aggregator @@ -109,8 +115,8 @@ func (p *DaemonImageProvider) trackSaveProgress() (*daemonProvideProgress, error } // pull a docker image -func (p *DaemonImageProvider) pull(ctx context.Context) error { - log.Debugf("pulling docker image=%q", p.imageStr) +func (p *daemonImageProvider) pull(ctx context.Context, client client.APIClient, imageRef string) error { + log.Debugf("pulling %s image=%q", p.name, imageRef) status := newPullStatus() defer func() { @@ -120,16 +126,16 @@ func (p *DaemonImageProvider) pull(ctx context.Context) error { // publish a pull event on the bus, allowing for read-only consumption of status bus.Publish(partybus.Event{ Type: event.PullDockerImage, - Source: p.imageStr, + Source: imageRef, Value: status, }) - options, err := p.pullOptions() + options, err := p.pullOptions(imageRef) if err != nil { return err } - resp, err := p.client.ImagePull(ctx, p.imageStr, options) + resp, err := client.ImagePull(ctx, imageRef, options) if err != nil { return fmt.Errorf("pull failed: %w", err) } @@ -156,7 +162,7 @@ func (p *DaemonImageProvider) pull(ctx context.Context) error { return nil } -func (p *DaemonImageProvider) pullOptions() (types.ImagePullOptions, error) { +func (p *daemonImageProvider) pullOptions(imageRef string) (types.ImagePullOptions, error) { options := types.ImagePullOptions{ Platform: p.platform.String(), } @@ -169,9 +175,9 @@ func (p *DaemonImageProvider) pullOptions() (types.ImagePullOptions, error) { log.Debugf("using docker config=%q", cfg.Filename) // get a URL that works with docker credential helpers - url, err := authURL(p.imageStr, true) + url, err := authURL(imageRef, true) if err != nil { - log.Warnf("failed to determine auth url from image=%q: %+v", p.imageStr, err) + log.Warnf("failed to determine auth url from image=%q: %+v", imageRef, err) return options, nil } @@ -187,9 +193,9 @@ func (p *DaemonImageProvider) pullOptions() (types.ImagePullOptions, error) { // docker credential helper was unnecessary (since the user isn't using a credential helper). For this reason // lets try this auth config lookup again, but this time for a url that doesn't consider the dockerhub // workaround for the credential helper. - url, err = authURL(p.imageStr, false) + url, err = authURL(imageRef, false) if err != nil { - log.Warnf("failed to determine auth url from image=%q: %+v", p.imageStr, err) + log.Warnf("failed to determine auth url from image=%q: %+v", imageRef, err) return options, nil } @@ -210,8 +216,8 @@ func (p *DaemonImageProvider) pullOptions() (types.ImagePullOptions, error) { return options, nil } -func authURL(imageStr string, dockerhubWorkaround bool) (string, error) { - ref, err := name.ParseReference(imageStr) +func authURL(imageRef string, dockerhubWorkaround bool) (string, error) { + ref, err := name.ParseReference(imageRef) if err != nil { return "", err } @@ -229,13 +235,33 @@ func authURL(imageStr string, dockerhubWorkaround bool) (string, error) { } // Provide an image object that represents the cached docker image tar fetched from a docker daemon. -func (p *DaemonImageProvider) Provide(ctx context.Context, userMetadata ...image.AdditionalMetadata) (*image.Image, error) { - if err := p.pullImageIfMissing(ctx); err != nil { +func (p *daemonImageProvider) Provide(ctx context.Context) (*image.Image, error) { + apiClient, err := p.newAPIClient() + if err != nil { + return nil, fmt.Errorf("%s not available: %w", p.name, err) + } + + defer func() { + if err := apiClient.Close(); err != nil { + log.Errorf("unable to close %s client: %+v", p.name, err) + } + }() + + c2, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + pong, err := apiClient.Ping(c2) + if err != nil || pong.APIVersion == "" { + return nil, fmt.Errorf("unable to get %s API response: %w", p.name, err) + } + + imageRef, err := p.pullImageIfMissing(ctx, apiClient) + if err != nil { return nil, err } // inspect the image that might have been pulled - inspectResult, _, err := p.client.ImageInspectWithRaw(ctx, p.imageStr) + inspectResult, _, err := apiClient.ImageInspectWithRaw(ctx, imageRef) if err != nil { return nil, fmt.Errorf("unable to inspect existing image: %w", err) } @@ -245,18 +271,19 @@ func (p *DaemonImageProvider) Provide(ctx context.Context, userMetadata ...image return nil, err } - tarFileName, err := p.saveImage(ctx) + tarFileName, err := p.saveImage(ctx, apiClient, imageRef) if err != nil { return nil, err } // use the existing tarball provider to process what was pulled from the docker daemon - return NewProviderFromTarball(tarFileName, p.tmpDirGen).Provide(ctx, withInspectMetadata(inspectResult, userMetadata)...) + return NewArchiveProvider(p.tmpDirGen, tarFileName, withInspectMetadata(inspectResult)...). + Provide(ctx) } -func (p *DaemonImageProvider) saveImage(ctx context.Context) (string, error) { +func (p *daemonImageProvider) saveImage(ctx context.Context, apiClient client.APIClient, imageRef string) (string, error) { // save the image from the docker daemon to a tar file - providerProgress, err := p.trackSaveProgress() + providerProgress, err := p.trackSaveProgress(ctx, apiClient, imageRef) if err != nil { return "", fmt.Errorf("unable to trace image save progress: %w", err) } @@ -267,7 +294,7 @@ func (p *DaemonImageProvider) saveImage(ctx context.Context) (string, error) { providerProgress.CopyProgress.SetComplete() }() - imageTempDir, err := p.tmpDirGen.NewDirectory("docker-daemon-image") + imageTempDir, err := p.tmpDirGen.NewDirectory(fmt.Sprintf("%s-daemon-image", p.name)) if err != nil { return "", err } @@ -284,8 +311,8 @@ func (p *DaemonImageProvider) saveImage(ctx context.Context) (string, error) { } }() - providerProgress.Stage.Set("requesting image from docker") - readCloser, err := p.client.ImageSave(ctx, []string{p.imageStr}) + providerProgress.Stage.Set(fmt.Sprintf("requesting image from %s", p.name)) + readCloser, err := apiClient.ImageSave(ctx, []string{imageRef}) if err != nil { return "", fmt.Errorf("unable to save image tar: %w", err) } @@ -316,36 +343,41 @@ func (p *DaemonImageProvider) saveImage(ctx context.Context) (string, error) { return tempTarFile.Name(), nil } -func (p *DaemonImageProvider) pullImageIfMissing(ctx context.Context) error { +func (p *daemonImageProvider) pullImageIfMissing(ctx context.Context, apiClient client.APIClient) (imageRef string, err error) { + imageRef, originalImageRef, err := image.ParseReference(p.imageStr) + if err != nil { + return "", err + } + // check if the image exists locally - inspectResult, _, err := p.client.ImageInspectWithRaw(ctx, p.imageStr) + inspectResult, _, err := apiClient.ImageInspectWithRaw(ctx, imageRef) if err != nil { - inspectResult, _, err = p.client.ImageInspectWithRaw(ctx, p.originalImageRef) + inspectResult, _, err = apiClient.ImageInspectWithRaw(ctx, originalImageRef) if err == nil { - p.imageStr = strings.TrimSuffix(p.imageStr, ":latest") + imageRef = strings.TrimSuffix(imageRef, ":latest") } } if err != nil { if client.IsErrNotFound(err) { - if err = p.pull(ctx); err != nil { - return err + if err = p.pull(ctx, apiClient, imageRef); err != nil { + return imageRef, err } } else { - return fmt.Errorf("unable to inspect existing image: %w", err) + return imageRef, fmt.Errorf("unable to inspect existing image: %w", err) } } else { // looks like the image exists, but if the platform doesn't match what the user specified, we may need to // pull the image again with the correct platform specifier, which will override the local tag. - if err := p.validatePlatform(inspectResult); err != nil { - if err = p.pull(ctx); err != nil { - return err + if err = p.validatePlatform(inspectResult); err != nil { + if err = p.pull(ctx, apiClient, imageRef); err != nil { + return imageRef, err } } } - return nil + return imageRef, nil } -func (p *DaemonImageProvider) validatePlatform(i types.ImageInspect) error { +func (p *daemonImageProvider) validatePlatform(i types.ImageInspect) error { if p.platform == nil { // the user did not specify a platform return nil @@ -364,16 +396,13 @@ func (p *DaemonImageProvider) validatePlatform(i types.ImageInspect) error { return nil } -func withInspectMetadata(i types.ImageInspect, userMetadata []image.AdditionalMetadata) (metadata []image.AdditionalMetadata) { +func withInspectMetadata(i types.ImageInspect) (metadata []image.AdditionalMetadata) { metadata = append(metadata, image.WithTags(i.RepoTags...), image.WithRepoDigests(i.RepoDigests...), image.WithArchitecture(i.Architecture, ""), // since we don't have variant info from the image directly, we don't report it image.WithOS(i.Os), ) - - // apply user-supplied metadata last to override any default behavior - metadata = append(metadata, userMetadata...) return metadata } diff --git a/pkg/image/docker/daemon_provider_test.go b/pkg/image/docker/daemon_provider_test.go index f2760604..ef30a56d 100644 --- a/pkg/image/docker/daemon_provider_test.go +++ b/pkg/image/docker/daemon_provider_test.go @@ -85,50 +85,3 @@ func Test_authURL(t *testing.T) { }) } } - -func TestNewProviderFromDaemon_imageParsing(t *testing.T) { - tests := []struct { - image string - want string - wantErr require.ErrorAssertionFunc - }{ - { - image: "alpine:sometag", - want: "alpine:sometag", - }, - { - image: "alpine:latest", - want: "alpine:latest", - }, - { - image: "alpine", - want: "alpine:latest", - }, - { - image: "registry.place.io/thing:version", - want: "registry.place.io/thing:version", - }, - { - image: "alpine@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", - want: "alpine@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", - }, - { - image: "alpine:sometag@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", - want: "alpine:sometag@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", - }, - } - for _, tt := range tests { - t.Run(tt.image, func(t *testing.T) { - if tt.wantErr == nil { - tt.wantErr = require.NoError - } - got, err := NewProviderFromDaemon(tt.image, nil, nil, nil) - tt.wantErr(t, err) - if err != nil { - return - } - require.NotNil(t, got) - assert.Equal(t, tt.want, got.imageStr) - }) - } -} diff --git a/pkg/image/docker/tarball_provider.go b/pkg/image/docker/tarball_provider.go index 0e5b6474..7819e717 100644 --- a/pkg/image/docker/tarball_provider.go +++ b/pkg/image/docker/tarball_provider.go @@ -13,24 +13,32 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) +const Archive image.Source = image.DockerTarballSource + +// NewArchiveProvider creates a new provider able to resolve docker tarball archives +func NewArchiveProvider(tmpDirGen *file.TempDirGenerator, path string, additionalMetadata ...image.AdditionalMetadata) image.Provider { + return &tarballImageProvider{ + tmpDirGen: tmpDirGen, + path: path, + additionalMetadata: additionalMetadata, + } +} + var ErrMultipleManifests = fmt.Errorf("cannot process multiple docker manifests") -// TarballImageProvider is a image.Provider for a docker image (V2) for an existing tar on disk (the output from a "docker image save ..." command). -type TarballImageProvider struct { - path string - tmpDirGen *file.TempDirGenerator +// tarballImageProvider is a image.Provider for a docker image (V2) for an existing tar on disk (the output from a "docker image save ..." command). +type tarballImageProvider struct { + tmpDirGen *file.TempDirGenerator + path string + additionalMetadata []image.AdditionalMetadata } -// NewProviderFromTarball creates a new provider instance for the specific image already at the given path. -func NewProviderFromTarball(path string, tmpDirGen *file.TempDirGenerator) *TarballImageProvider { - return &TarballImageProvider{ - path: path, - tmpDirGen: tmpDirGen, - } +func (p *tarballImageProvider) Name() string { + return Archive } // Provide an image object that represents the docker image tar at the configured location on disk. -func (p *TarballImageProvider) Provide(_ context.Context, userMetadata ...image.AdditionalMetadata) (*image.Image, error) { +func (p *tarballImageProvider) Provide(_ context.Context) (*image.Image, error) { img, err := tarball.ImageFromPath(p.path, nil) if err != nil { // raise a more controlled error for when there are multiple images within the given tar (from https://github.com/anchore/grype/issues/215) @@ -76,12 +84,17 @@ func (p *TarballImageProvider) Provide(_ context.Context, userMetadata ...image. } // apply user-supplied metadata last to override any default behavior - metadata = append(metadata, userMetadata...) + metadata = append(metadata, p.additionalMetadata...) contentTempDir, err := p.tmpDirGen.NewDirectory("docker-tarball-image") if err != nil { return nil, err } - return image.New(img, p.tmpDirGen, contentTempDir, metadata...), nil + out := image.New(img, p.tmpDirGen, contentTempDir, metadata...) + err = out.Read() + if err != nil { + return nil, err + } + return out, err } diff --git a/pkg/image/oci/directory_provider.go b/pkg/image/oci/directory_provider.go index 549d5870..422fdae1 100644 --- a/pkg/image/oci/directory_provider.go +++ b/pkg/image/oci/directory_provider.go @@ -11,22 +11,28 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -// DirectoryImageProvider is an image.Provider for an OCI image (V1) for an existing tar on disk (from a buildah push oci: command). -type DirectoryImageProvider struct { - path string - tmpDirGen *file.TempDirGenerator -} +const Directory image.Source = image.OciDirectorySource -// NewProviderFromPath creates a new provider instance for the specific image already at the given path. -func NewProviderFromPath(path string, tmpDirGen *file.TempDirGenerator) *DirectoryImageProvider { - return &DirectoryImageProvider{ - path: path, +// NewDirectoryProvider creates a new provider instance for the specific image already at the given path. +func NewDirectoryProvider(tmpDirGen *file.TempDirGenerator, path string) image.Provider { + return &directoryImageProvider{ tmpDirGen: tmpDirGen, + path: path, } } +// directoryImageProvider is an image.Provider for an OCI image (V1) for an existing tar on disk (from a buildah push oci: command). +type directoryImageProvider struct { + tmpDirGen *file.TempDirGenerator + path string +} + +func (p *directoryImageProvider) Name() string { + return Directory +} + // Provide an image object that represents the OCI image as a directory. -func (p *DirectoryImageProvider) Provide(_ context.Context, userMetadata ...image.AdditionalMetadata) (*image.Image, error) { +func (p *directoryImageProvider) Provide(_ context.Context) (*image.Image, error) { pathObj, err := layout.FromPath(p.path) if err != nil { return nil, fmt.Errorf("unable to read image from OCI directory path %q: %w", p.path, err) @@ -69,15 +75,17 @@ func (p *DirectoryImageProvider) Provide(_ context.Context, userMetadata ...imag metadata = append(metadata, image.WithManifest(rawManifest)) } - // apply user-supplied metadata last to override any default behavior - metadata = append(metadata, userMetadata...) - contentTempDir, err := p.tmpDirGen.NewDirectory("oci-dir-image") if err != nil { return nil, err } - return image.New(img, p.tmpDirGen, contentTempDir, metadata...), nil + out := image.New(img, p.tmpDirGen, contentTempDir, metadata...) + err = out.Read() + if err != nil { + return nil, err + } + return out, err } func checkManifestDigestsEqual(manifests []v1.Descriptor) bool { diff --git a/pkg/image/oci/directory_provider_test.go b/pkg/image/oci/directory_provider_test.go index f4fe8c8b..f19098b1 100644 --- a/pkg/image/oci/directory_provider_test.go +++ b/pkg/image/oci/directory_provider_test.go @@ -1,6 +1,7 @@ package oci import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -12,16 +13,17 @@ func Test_NewProviderFromPath(t *testing.T) { //GIVEN path := "path" generator := file.TempDirGenerator{} + defer generator.Cleanup() //WHEN - provider := NewProviderFromPath(path, &generator) + provider := NewDirectoryProvider(&generator, path).(*directoryImageProvider) //THEN assert.NotNil(t, provider.path) assert.NotNil(t, provider.tmpDirGen) } -func Test_Directory_Provide(t *testing.T) { +func Test_Directory_Provider(t *testing.T) { //GIVEN tests := []struct { name string @@ -29,17 +31,20 @@ func Test_Directory_Provide(t *testing.T) { expectedErr bool }{ {"fails to read from path", "", true}, - {"reads invalid oci manifest", "test-fixtures/invalid_file", true}, - {"reads valid oci manifest with no images", "test-fixtures/no_manifests", true}, - {"reads a fully correct manifest", "test-fixtures/valid_manifest", false}, - {"reads a fully correct manifest with equal digests", "test-fixtures/valid_manifest", false}, + {"fails to read invalid oci manifest", "test-fixtures/invalid_file", true}, + {"fails to read valid oci manifest with no images", "test-fixtures/no_manifests", true}, + {"fails to read an invalid oci directory", "test-fixtures/valid_manifest", true}, + {"reads a valid oci directory", "test-fixtures/valid_oci_dir", false}, } + tmpDirGen := file.NewTempDirGenerator("tempDir") + defer tmpDirGen.Cleanup() + for _, tc := range tests { - provider := NewProviderFromPath(tc.path, file.NewTempDirGenerator("tempDir")) + provider := NewDirectoryProvider(tmpDirGen, tc.path) t.Run(tc.name, func(t *testing.T) { //WHEN - image, err := provider.Provide(nil) + image, err := provider.Provide(context.Background()) //THEN if tc.expectedErr { diff --git a/pkg/image/oci/registry_provider.go b/pkg/image/oci/registry_provider.go index 208c4a24..29f2f75f 100644 --- a/pkg/image/oci/registry_provider.go +++ b/pkg/image/oci/registry_provider.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "fmt" "net/http" + "runtime" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" @@ -16,26 +17,32 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -// RegistryImageProvider is an image.Provider capable of fetching and representing a container image fetched from a remote registry (described by the OCI distribution spec). -type RegistryImageProvider struct { - imageStr string - tmpDirGen *file.TempDirGenerator - registryOptions image.RegistryOptions - platform *image.Platform -} +const Registry image.Source = image.OciRegistrySource -// NewProviderFromRegistry creates a new provider instance for a specific image that will later be cached to the given directory. -func NewProviderFromRegistry(imgStr string, tmpDirGen *file.TempDirGenerator, registryOptions image.RegistryOptions, platform *image.Platform) *RegistryImageProvider { - return &RegistryImageProvider{ - imageStr: imgStr, +// NewRegistryProvider creates a new provider instance for a specific image that will later be cached to the given directory. +func NewRegistryProvider(tmpDirGen *file.TempDirGenerator, registryOptions image.RegistryOptions, imageStr string, platform *image.Platform) image.Provider { + return ®istryImageProvider{ tmpDirGen: tmpDirGen, - registryOptions: registryOptions, + imageStr: imageStr, platform: platform, + registryOptions: registryOptions, } } +// registryImageProvider is an image.Provider capable of fetching and representing a container image fetched from a remote registry (described by the OCI distribution spec). +type registryImageProvider struct { + tmpDirGen *file.TempDirGenerator + imageStr string + platform *image.Platform + registryOptions image.RegistryOptions +} + +func (p *registryImageProvider) Name() string { + return Registry +} + // Provide an image object that represents the cached docker image tar fetched a registry. -func (p *RegistryImageProvider) Provide(ctx context.Context, userMetadata ...image.AdditionalMetadata) (*image.Image, error) { +func (p *registryImageProvider) Provide(ctx context.Context) (*image.Image, error) { log.Debugf("pulling image info directly from registry image=%q", p.imageStr) imageTempDir, err := p.tmpDirGen.NewDirectory("oci-registry-image") @@ -48,7 +55,9 @@ func (p *RegistryImageProvider) Provide(ctx context.Context, userMetadata ...ima return nil, fmt.Errorf("unable to parse registry reference=%q: %+v", p.imageStr, err) } - options := prepareRemoteOptions(ctx, ref, p.registryOptions, p.platform) + platform := defaultPlatformIfNil(p.platform) + + options := prepareRemoteOptions(ctx, ref, p.registryOptions, platform) descriptor, err := remote.Get(ref, options...) if err != nil { @@ -73,17 +82,19 @@ func (p *RegistryImageProvider) Provide(ctx context.Context, userMetadata ...ima metadata = append(metadata, image.WithManifest(manifestBytes)) } - if p.platform != nil { + if platform != nil { metadata = append(metadata, - image.WithArchitecture(p.platform.Architecture, p.platform.Variant), - image.WithOS(p.platform.OS), + image.WithArchitecture(platform.Architecture, platform.Variant), + image.WithOS(platform.OS), ) } - // apply user-supplied metadata last to override any default behavior - metadata = append(metadata, userMetadata...) - - return image.New(img, p.tmpDirGen, imageTempDir, metadata...), nil + out := image.New(img, p.tmpDirGen, imageTempDir, metadata...) + err = out.Read() + if err != nil { + return nil, err + } + return out, err } func prepareReferenceOptions(registryOptions image.RegistryOptions) []name.Option { @@ -139,3 +150,19 @@ func getTransport(tlsConfig *tls.Config) *http.Transport { transport.TLSClientConfig = tlsConfig return transport } + +// defaultPlatformIfNil sets the platform to use the host's architecture +// if no platform was specified. The OCI registry NewProvider uses "linux/amd64" +// as a hard-coded default platform, which has surprised customers +// running stereoscope on non-amd64 hosts. If platform is already +// set on the config, or the code can't generate a matching platform, +// do nothing. +func defaultPlatformIfNil(platform *image.Platform) *image.Platform { + if platform == nil { + p, err := image.NewPlatform(fmt.Sprintf("linux/%s", runtime.GOARCH)) + if err == nil { + return p + } + } + return platform +} diff --git a/pkg/image/oci/registry_provider_test.go b/pkg/image/oci/registry_provider_test.go index 3c95f001..5df09cd6 100644 --- a/pkg/image/oci/registry_provider_test.go +++ b/pkg/image/oci/registry_provider_test.go @@ -2,28 +2,53 @@ package oci import ( "context" + "fmt" "net/http" + "net/http/httptest" "reflect" + "strings" "testing" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" ) +func Test_RegistryProvider(t *testing.T) { + imageName := "my-image" + imageTag := "the-tag" + + registryHost := makeRegistry(t) + pushRandomRegistryImage(t, registryHost, imageName, imageTag) + + generator := file.TempDirGenerator{} + defer generator.Cleanup() + + options := image.RegistryOptions{} + provider := NewRegistryProvider(&generator, options, fmt.Sprintf("%s/%s:%s", registryHost, imageName, imageTag), nil) + img, err := provider.Provide(context.TODO()) + assert.NoError(t, err) + assert.NotNil(t, img) +} + func Test_NewProviderFromRegistry(t *testing.T) { //GIVEN imageStr := "image" generator := file.TempDirGenerator{} + defer generator.Cleanup() options := image.RegistryOptions{} platform := &image.Platform{} //WHEN - provider := NewProviderFromRegistry(imageStr, &generator, options, platform) + provider := NewRegistryProvider(&generator, options, imageStr, platform).(*registryImageProvider) //THEN assert.NotNil(t, provider.imageStr) @@ -36,6 +61,7 @@ func Test_Registry_Provide_FailsUnauthorized(t *testing.T) { //GIVEN imageStr := "image" generator := file.TempDirGenerator{} + defer generator.Cleanup() options := image.RegistryOptions{ InsecureSkipTLSVerify: true, Credentials: []image.RegistryCredentials{ @@ -46,7 +72,7 @@ func Test_Registry_Provide_FailsUnauthorized(t *testing.T) { }, } platform := &image.Platform{} - provider := NewProviderFromRegistry(imageStr, &generator, options, platform) + provider := NewRegistryProvider(&generator, options, imageStr, platform) ctx := context.Background() //WHEN @@ -61,11 +87,12 @@ func Test_Registry_Provide_FailsImageMissingPlatform(t *testing.T) { //GIVEN imageStr := "docker.io/golang:1.18" generator := file.TempDirGenerator{} + defer generator.Cleanup() options := image.RegistryOptions{ InsecureSkipTLSVerify: true, } platform := &image.Platform{} - provider := NewProviderFromRegistry(imageStr, &generator, options, platform) + provider := NewRegistryProvider(&generator, options, imageStr, platform) ctx := context.Background() //WHEN @@ -76,10 +103,11 @@ func Test_Registry_Provide_FailsImageMissingPlatform(t *testing.T) { assert.Error(t, err) } -func Test_Registry_Provide(t *testing.T) { +func Test_DockerMainRegistry_Provide(t *testing.T) { //GIVEN - imageStr := "golang:1.18" + imageStr := "alpine:3.17" generator := file.TempDirGenerator{} + defer generator.Cleanup() options := image.RegistryOptions{ InsecureSkipTLSVerify: true, } @@ -87,7 +115,7 @@ func Test_Registry_Provide(t *testing.T) { OS: "linux", Architecture: "amd64", } - provider := NewProviderFromRegistry(imageStr, &generator, options, platform) + provider := NewRegistryProvider(&generator, options, imageStr, platform) ctx := context.Background() //WHEN @@ -147,3 +175,33 @@ func Test_getTransport_haxProxyCfg(t *testing.T) { t.Errorf("unexpected proxy config (-want +got):\n%s", d) } } + +func pushRandomRegistryImage(t *testing.T, registryHost, repo, tag string) { + t.Helper() + + repoTag := repo + ":" + tag + + img, err := random.Image(1024, 2) + require.NoError(t, err) + + opts := []name.Option{name.Insecure, name.WithDefaultRegistry(registryHost)} + ref, err := name.ParseReference(repoTag, opts...) + require.NoError(t, err) + + remoteopts := []remote.Option{remote.WithUserAgent("syft-test-util")} + err = remote.Write(ref, img, remoteopts...) + require.NoError(t, err) + + latestTag, err := name.NewTag(tag, opts...) + require.NoError(t, err) + err = remote.Tag(latestTag, img, remoteopts...) + require.NoError(t, err) +} + +func makeRegistry(t *testing.T) (registryHost string) { + memoryBlobHandler := registry.NewInMemoryBlobHandler() + registryInstance := registry.New(registry.WithBlobHandler(memoryBlobHandler)) + ts := httptest.NewServer(http.HandlerFunc(registryInstance.ServeHTTP)) + t.Cleanup(ts.Close) + return strings.TrimPrefix(ts.URL, "http://") +} diff --git a/pkg/image/oci/tarball_provider.go b/pkg/image/oci/tarball_provider.go index 74a29278..19b1e227 100644 --- a/pkg/image/oci/tarball_provider.go +++ b/pkg/image/oci/tarball_provider.go @@ -9,22 +9,28 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -// TarballImageProvider is an image.Provider for an OCI image (V1) for an existing tar on disk (from a buildah push oci-archive:.tar command). -type TarballImageProvider struct { - path string - tmpDirGen *file.TempDirGenerator -} +const Archive image.Source = image.OciTarballSource -// NewProviderFromTarball creates a new provider instance for the specific image tarball already at the given path. -func NewProviderFromTarball(path string, tmpDirGen *file.TempDirGenerator) *TarballImageProvider { - return &TarballImageProvider{ - path: path, +// NewArchiveProvider creates a new provider instance for the specific image tarball already at the given path. +func NewArchiveProvider(tmpDirGen *file.TempDirGenerator, path string) image.Provider { + return &tarballImageProvider{ tmpDirGen: tmpDirGen, + path: path, } } +// tarballImageProvider is an image.Provider for an OCI image (V1) for an existing tar on disk (from a buildah push oci-archive:.tar command). +type tarballImageProvider struct { + tmpDirGen *file.TempDirGenerator + path string +} + +func (p *tarballImageProvider) Name() string { + return Archive +} + // Provide an image object that represents the OCI image from a tarball. -func (p *TarballImageProvider) Provide(ctx context.Context, metadata ...image.AdditionalMetadata) (*image.Image, error) { +func (p *tarballImageProvider) Provide(ctx context.Context) (*image.Image, error) { // note: we are untaring the image and using the existing directory provider, we could probably enhance the google // container registry lib to do this without needing to untar to a temp dir (https://github.com/google/go-containerregistry/issues/726) f, err := os.Open(p.path) @@ -41,5 +47,5 @@ func (p *TarballImageProvider) Provide(ctx context.Context, metadata ...image.Ad return nil, err } - return NewProviderFromPath(tempDir, p.tmpDirGen).Provide(ctx, metadata...) + return NewDirectoryProvider(p.tmpDirGen, tempDir).Provide(ctx) } diff --git a/pkg/image/oci/tarball_provider_test.go b/pkg/image/oci/tarball_provider_test.go index ae2707ca..126e9b48 100644 --- a/pkg/image/oci/tarball_provider_test.go +++ b/pkg/image/oci/tarball_provider_test.go @@ -1,6 +1,7 @@ package oci import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -12,9 +13,10 @@ func Test_NewProviderFromTarball(t *testing.T) { //GIVEN path := "path" generator := file.TempDirGenerator{} + defer generator.Cleanup() //WHEN - provider := NewProviderFromTarball(path, &generator) + provider := NewArchiveProvider(&generator, path).(*tarballImageProvider) //THEN assert.NotNil(t, provider.path) @@ -23,10 +25,13 @@ func Test_NewProviderFromTarball(t *testing.T) { func Test_TarballProvide(t *testing.T) { //GIVEN - provider := NewProviderFromTarball("test-fixtures/file.tar", file.NewTempDirGenerator("tempDir")) + generator := file.NewTempDirGenerator("tempDir") + defer generator.Cleanup() + + provider := NewArchiveProvider(generator, "test-fixtures/valid-oci.tar") //WHEN - image, err := provider.Provide(nil) + image, err := provider.Provide(context.TODO()) //THEN assert.NoError(t, err) @@ -35,10 +40,13 @@ func Test_TarballProvide(t *testing.T) { func Test_TarballProvide_Fails(t *testing.T) { //GIVEN - provider := NewProviderFromTarball("", file.NewTempDirGenerator("tempDir")) + generator := file.NewTempDirGenerator("tempDir") + defer generator.Cleanup() + + provider := NewArchiveProvider(generator, "") //WHEN - image, err := provider.Provide(nil) + image, err := provider.Provide(context.TODO()) //THEN assert.Error(t, err) diff --git a/pkg/image/oci/test-fixtures/valid-oci.tar b/pkg/image/oci/test-fixtures/valid-oci.tar new file mode 100644 index 00000000..4785baa5 Binary files /dev/null and b/pkg/image/oci/test-fixtures/valid-oci.tar differ diff --git a/pkg/image/oci/test-fixtures/valid_oci_dir/blobs/sha256/00ffd085e9e7c06d20fdc61119a8d08e5c8bd3c1c320d10494ce6ed86691c06c b/pkg/image/oci/test-fixtures/valid_oci_dir/blobs/sha256/00ffd085e9e7c06d20fdc61119a8d08e5c8bd3c1c320d10494ce6ed86691c06c new file mode 100644 index 00000000..39e5b9b1 Binary files /dev/null and b/pkg/image/oci/test-fixtures/valid_oci_dir/blobs/sha256/00ffd085e9e7c06d20fdc61119a8d08e5c8bd3c1c320d10494ce6ed86691c06c differ diff --git a/pkg/image/oci/test-fixtures/valid_oci_dir/blobs/sha256/61ee19e869a529075634f762c1eb191962777e0803598758cbf076edfadfb046 b/pkg/image/oci/test-fixtures/valid_oci_dir/blobs/sha256/61ee19e869a529075634f762c1eb191962777e0803598758cbf076edfadfb046 new file mode 100644 index 00000000..75615d11 --- /dev/null +++ b/pkg/image/oci/test-fixtures/valid_oci_dir/blobs/sha256/61ee19e869a529075634f762c1eb191962777e0803598758cbf076edfadfb046 @@ -0,0 +1 @@ +{"created":"2024-02-07T09:02:36.826417009Z","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"WorkingDir":"/"},"rootfs":{"type":"layers","diff_ids":["sha256:7cec5b0994b7ef959705617ab9c8bbd495446c4a80afa28aba0490f16772c9b3"]},"history":[{"created":"2024-02-07T09:02:36.826417009Z","created_by":"COPY README.md / # buildkit","comment":"buildkit.dockerfile.v0"}]} \ No newline at end of file diff --git a/pkg/image/oci/test-fixtures/valid_oci_dir/blobs/sha256/c1ed04a3da941a5dd09b58b16c37f065557863d382ef97995ddac885a8452ebb b/pkg/image/oci/test-fixtures/valid_oci_dir/blobs/sha256/c1ed04a3da941a5dd09b58b16c37f065557863d382ef97995ddac885a8452ebb new file mode 100644 index 00000000..7547a08f --- /dev/null +++ b/pkg/image/oci/test-fixtures/valid_oci_dir/blobs/sha256/c1ed04a3da941a5dd09b58b16c37f065557863d382ef97995ddac885a8452ebb @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:61ee19e869a529075634f762c1eb191962777e0803598758cbf076edfadfb046","size":433},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:00ffd085e9e7c06d20fdc61119a8d08e5c8bd3c1c320d10494ce6ed86691c06c","size":120}]} \ No newline at end of file diff --git a/pkg/image/oci/test-fixtures/valid_oci_dir/index.json b/pkg/image/oci/test-fixtures/valid_oci_dir/index.json new file mode 100644 index 00000000..cf07e3a3 --- /dev/null +++ b/pkg/image/oci/test-fixtures/valid_oci_dir/index.json @@ -0,0 +1,25 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:c1ed04a3da941a5dd09b58b16c37f065557863d382ef97995ddac885a8452ebb", + "size": 401, + "annotations": { + "org.opencontainers.image.ref.name": "latest" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:c1ed04a3da941a5dd09b58b16c37f065557863d382ef97995ddac885a8452ebb", + "size": 401, + "annotations": { + "org.opencontainers.image.ref.name": "1.0" + }, + "platform": { + "architecture": "amd64", + "os": "linux" + } + } + ] +} \ No newline at end of file diff --git a/pkg/image/oci/test-fixtures/valid_oci_dir/oci-layout b/pkg/image/oci/test-fixtures/valid_oci_dir/oci-layout new file mode 100644 index 00000000..1343d370 --- /dev/null +++ b/pkg/image/oci/test-fixtures/valid_oci_dir/oci-layout @@ -0,0 +1 @@ +{"imageLayoutVersion":"1.0.0"} \ No newline at end of file diff --git a/pkg/image/parse_reference.go b/pkg/image/parse_reference.go new file mode 100644 index 00000000..6019ded8 --- /dev/null +++ b/pkg/image/parse_reference.go @@ -0,0 +1,16 @@ +package image + +import "github.com/google/go-containerregistry/pkg/name" + +func ParseReference(imageStr string) (imageRef string, originalRef string, err error) { + ref, err := name.ParseReference(imageStr, name.WithDefaultRegistry("")) + if err != nil { + return "", "", err + } + tag, ok := ref.(name.Tag) + if ok { + imageStr = tag.Name() + originalRef = tag.String() // blindly takes the original input passed into Tag + } + return imageStr, originalRef, nil +} diff --git a/pkg/image/parse_reference_test.go b/pkg/image/parse_reference_test.go new file mode 100644 index 00000000..f28d43aa --- /dev/null +++ b/pkg/image/parse_reference_test.go @@ -0,0 +1,58 @@ +package image + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewProviderFromDaemon_ParseReference(t *testing.T) { + tests := []struct { + image string + want string + wantErr require.ErrorAssertionFunc + }{ + { + image: "alpine:sometag", + want: "alpine:sometag", + }, + { + image: "alpine:latest", + want: "alpine:latest", + }, + { + image: "alpine", + want: "alpine:latest", + }, + { + image: "registry.place.io/thing:version", + want: "registry.place.io/thing:version", + }, + { + image: "alpine@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", + want: "alpine@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", + }, + { + image: "alpine:sometag@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", + want: "alpine:sometag@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", + }, + { + image: "some:invalid:tag", + wantErr: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.image, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + got, _, err := ParseReference(tt.image) + tt.wantErr(t, err) + if err != nil { + return + } + require.NotNil(t, got) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/image/podman/daemon_provider.go b/pkg/image/podman/daemon_provider.go new file mode 100644 index 00000000..16b938bd --- /dev/null +++ b/pkg/image/podman/daemon_provider.go @@ -0,0 +1,18 @@ +package podman + +import ( + "github.com/docker/docker/client" + + "github.com/anchore/stereoscope/internal/podman" + "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/stereoscope/pkg/image/docker" +) + +const Daemon image.Source = image.PodmanDaemonSource + +func NewDaemonProvider(tmpDirGen *file.TempDirGenerator, imageStr string, platform *image.Platform) image.Provider { + return docker.NewAPIClientProvider(Daemon, tmpDirGen, imageStr, platform, func() (client.APIClient, error) { + return podman.GetClient() + }) +} diff --git a/pkg/image/provider.go b/pkg/image/provider.go index a546cb32..a3c70e2c 100644 --- a/pkg/image/provider.go +++ b/pkg/image/provider.go @@ -1,9 +1,12 @@ package image -import "context" +import ( + "context" +) // Provider is an abstraction for any object that provides image objects (e.g. the docker daemon API, a tar file of // an OCI image, podman varlink API, etc.). type Provider interface { - Provide(context.Context, ...AdditionalMetadata) (*Image, error) + Name() string + Provide(context.Context) (*Image, error) } diff --git a/pkg/image/sif/provider.go b/pkg/image/sif/archive_provider.go similarity index 62% rename from pkg/image/sif/provider.go rename to pkg/image/sif/archive_provider.go index 2d81b838..e8bf80b0 100644 --- a/pkg/image/sif/provider.go +++ b/pkg/image/sif/archive_provider.go @@ -9,23 +9,29 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) -// SingularityImageProvider is an image.Provider for a Singularity Image Format (SIF) image. -type SingularityImageProvider struct { - path string - tmpDirGen *file.TempDirGenerator -} +const ProviderName = image.SingularitySource -// NewProviderFromPath creates a new provider instance for the Singularity Image Format (SIF) image +// NewArchiveProvider creates a new provider instance for the Singularity Image Format (SIF) image // at path. -func NewProviderFromPath(path string, tmpDirGen *file.TempDirGenerator) *SingularityImageProvider { - return &SingularityImageProvider{ - path: path, +func NewArchiveProvider(tmpDirGen *file.TempDirGenerator, path string) image.Provider { + return &singularityImageProvider{ tmpDirGen: tmpDirGen, + path: path, } } +// singularityImageProvider is an image.Provider for a Singularity Image Format (SIF) image. +type singularityImageProvider struct { + tmpDirGen *file.TempDirGenerator + path string +} + +func (p *singularityImageProvider) Name() string { + return ProviderName +} + // Provide returns an Image that represents a Singularity Image Format (SIF) image. -func (p *SingularityImageProvider) Provide(_ context.Context, userMetadata ...image.AdditionalMetadata) (*image.Image, error) { +func (p *singularityImageProvider) Provide(_ context.Context) (*image.Image, error) { // We need to map the SIF to a GGCR v1.Image. Start with an implementation of the GGCR // partial.UncompressedImageCore interface. si, err := newSIFImage(p.path) @@ -50,7 +56,11 @@ func (p *SingularityImageProvider) Provide(_ context.Context, userMetadata ...im image.WithOS("linux"), image.WithArchitecture(si.arch, ""), } - metadata = append(metadata, userMetadata...) - return image.New(ui, p.tmpDirGen, contentCacheDir, metadata...), nil + out := image.New(ui, p.tmpDirGen, contentCacheDir, metadata...) + err = out.Read() + if err != nil { + return nil, err + } + return out, err } diff --git a/pkg/image/sif/provider_test.go b/pkg/image/sif/archive_provider_test.go similarity index 74% rename from pkg/image/sif/provider_test.go rename to pkg/image/sif/archive_provider_test.go index 0c3901e1..e0cc43cc 100644 --- a/pkg/image/sif/provider_test.go +++ b/pkg/image/sif/archive_provider_test.go @@ -9,15 +9,13 @@ import ( "github.com/sylabs/sif/v2/pkg/sif" "github.com/anchore/stereoscope/pkg/file" - "github.com/anchore/stereoscope/pkg/image" ) func TestSingularityImageProvider_Provide(t *testing.T) { tests := []struct { - name string - path string - userMetadata []image.AdditionalMetadata - wantErr error + name string + path string + wantErr error }{ { name: "NoObjects", @@ -35,9 +33,9 @@ func TestSingularityImageProvider_Provide(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewProviderFromPath(tt.path, file.NewTempDirGenerator("")) + p := NewArchiveProvider(file.NewTempDirGenerator(""), tt.path) - i, err := p.Provide(context.Background(), tt.userMetadata...) + i, err := p.Provide(context.Background()) t.Cleanup(func() { _ = i.Cleanup() }) if got, want := err, tt.wantErr; !errors.Is(got, want) { diff --git a/pkg/image/source.go b/pkg/image/source.go index d38668ad..106c0cb9 100644 --- a/pkg/image/source.go +++ b/pkg/image/source.go @@ -1,268 +1,15 @@ package image -import ( - "context" - "errors" - "fmt" - "io" - "os" - "path" - "strings" - "time" - - "github.com/google/go-containerregistry/pkg/name" - "github.com/mitchellh/go-homedir" - "github.com/spf13/afero" - "github.com/sylabs/sif/v2/pkg/sif" - - "github.com/anchore/stereoscope/internal/containerd" - "github.com/anchore/stereoscope/internal/docker" - "github.com/anchore/stereoscope/internal/podman" - "github.com/anchore/stereoscope/pkg/file" -) +type Source = string const ( - UnknownSource Source = iota - ContainerdDaemonSource - DockerTarballSource - DockerDaemonSource - OciDirectorySource - OciTarballSource - OciRegistrySource - PodmanDaemonSource - SingularitySource + UnknownSource Source = "" + ContainerdDaemonSource Source = "containerd" + DockerTarballSource Source = "docker-archive" + DockerDaemonSource Source = "docker" + OciDirectorySource Source = "oci-dir" + OciTarballSource Source = "oci-archive" + OciRegistrySource Source = "oci-registry" + PodmanDaemonSource Source = "podman" + SingularitySource Source = "singularity" ) - -const SchemeSeparator = ":" - -var sourceStr = [...]string{ - "UnknownSource", - "ContainerdDaemon", - "DockerTarball", - "DockerDaemon", - "OciDirectory", - "OciTarball", - "OciRegistry", - "PodmanDaemon", - "Singularity", -} - -var AllSources = []Source{ - ContainerdDaemonSource, - DockerTarballSource, - DockerDaemonSource, - OciDirectorySource, - OciTarballSource, - OciRegistrySource, - PodmanDaemonSource, - SingularitySource, -} - -// Source is a concrete a selection of valid concrete image providers. -type Source uint8 - -// isRegistryReference takes a string and indicates if it conforms to a container image reference. -func isRegistryReference(imageSpec string) bool { - // note: strict validation requires there to be a default registry (e.g. docker.io) which we cannot assume will be provided - // we only want to validate the bare minimum number of image specification features, not exhaustive. - _, err := name.ParseReference(imageSpec, name.WeakValidation) - return err == nil -} - -// ParseSourceScheme attempts to resolve a concrete image source selection from a scheme in a user string. -func ParseSourceScheme(source string) Source { - source = strings.ToLower(source) - switch source { - case "containerd": - return ContainerdDaemonSource - case "docker-archive": - return DockerTarballSource - case "docker": - return DockerDaemonSource - case "podman": - return PodmanDaemonSource - case "oci-dir": - return OciDirectorySource - case "oci-archive": - return OciTarballSource - case "oci-registry", "registry": - return OciRegistrySource - case "singularity": - return SingularitySource - } - return UnknownSource -} - -// DetectSource takes a user string and determines the image source (e.g. the docker daemon, a tar file, etc.) returning the string subset representing the image (or nothing if it is unknown). -// note: parsing is done relative to the given string and environmental evidence (i.e. the given filesystem) to determine the actual source. -func DetectSource(userInput string) (Source, string, error) { - return detectSource(afero.NewOsFs(), userInput) -} - -// DetectSource takes a user string and determines the image source (e.g. the docker daemon, a tar file, etc.) returning the string subset representing the image (or nothing if it is unknown). -// note: parsing is done relative to the given string and environmental evidence (i.e. the given filesystem) to determine the actual source. -func detectSource(fs afero.Fs, userInput string) (Source, string, error) { - candidates := strings.SplitN(userInput, SchemeSeparator, 2) - - var source = UnknownSource - var location = userInput - var sourceHint string - var err error - if len(candidates) == 2 { - // the user may have provided a source hint (or this is a split from a path or docker image reference, we aren't certain yet) - sourceHint = candidates[0] - source = ParseSourceScheme(sourceHint) - } - if source != UnknownSource { - // if we found source from hint, than remove the hint from the location - location = strings.TrimPrefix(userInput, sourceHint+SchemeSeparator) - } else { - // a valid source hint wasnt provided/detected, try detect one - source, err = detectSourceFromPath(fs, location) - if err != nil { - return UnknownSource, "", err - } - } - - switch source { - case OciDirectorySource, OciTarballSource, DockerTarballSource, SingularitySource: - // since the scheme was explicitly given, that means that home dir tilde expansion would not have been done by the shell (so we have to) - location, err = homedir.Expand(location) - if err != nil { - return UnknownSource, "", fmt.Errorf("unable to expand potential home dir expression: %w", err) - } - case UnknownSource: - location = "" - } - - return source, location, nil -} - -// DetermineDefaultImagePullSource takes an image reference string as input, and -// determines a Source to use to pull the image. If the input doesn't specify an -// image reference (i.e. an image that can be _pulled_), UnknownSource is -// returned. Otherwise, if the Docker daemon is available, DockerDaemonSource is -// returned, and if not, OciRegistrySource is returned. -func DetermineDefaultImagePullSource(userInput string) Source { - if !isRegistryReference(userInput) { - return UnknownSource - } - - // verify that the Docker daemon is accessible before assuming we can use it - c, err := docker.GetClient() - if err == nil { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - pong, err := c.Ping(ctx) - if err == nil && pong.APIVersion != "" { - // the Docker daemon exists and is accessible - return DockerDaemonSource - } - } - - c, err = podman.GetClient() - if err == nil { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - pong, err := c.Ping(ctx) - if err == nil && pong.APIVersion != "" { - // the Docker daemon exists and is accessible - return PodmanDaemonSource - } - } - - cd, err := containerd.GetClient() - if err == nil { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - pong, err := cd.Version(ctx) - if err == nil && pong.Version != "" { - // the Docker daemon exists and is accessible - return ContainerdDaemonSource - } - } - - // fallback to using the registry directly - return OciRegistrySource -} - -// DetectSourceFromPath will distinguish between a oci-layout dir, oci-archive, and a docker-archive for a given filesystem. -func DetectSourceFromPath(imgPath string) (Source, error) { - return detectSourceFromPath(afero.NewOsFs(), imgPath) -} - -// detectSourceFromPath will distinguish between a oci-layout dir, oci-archive, and a docker-archive for a given filesystem. -func detectSourceFromPath(fs afero.Fs, imgPath string) (Source, error) { - imgPath, err := homedir.Expand(imgPath) - if err != nil { - return UnknownSource, fmt.Errorf("unable to expand potential home dir expression: %w", err) - } - - pathStat, err := fs.Stat(imgPath) - if os.IsNotExist(err) { - return UnknownSource, nil - } else if err != nil { - return UnknownSource, fmt.Errorf("failed to open path=%s: %w", imgPath, err) - } - - if pathStat.IsDir() { - // check for oci-directory - if _, err := fs.Stat(path.Join(imgPath, "oci-layout")); !os.IsNotExist(err) { - return OciDirectorySource, nil - } - - // there are no other directory-based source formats supported - return UnknownSource, nil - } - - f, err := fs.Open(imgPath) - if err != nil { - return UnknownSource, fmt.Errorf("unable to open file=%s: %w", imgPath, err) - } - defer f.Close() - - // Check for Singularity container. - fi, err := sif.LoadContainer(f, sif.OptLoadWithCloseOnUnload(false)) - if err == nil { - return SingularitySource, fi.UnloadContainer() - } - - // assume this is an archive... - for _, pair := range []struct { - path string - source Source - }{ - { - "manifest.json", - DockerTarballSource, - }, - { - "oci-layout", - OciTarballSource, - }, - } { - if _, err = f.Seek(0, io.SeekStart); err != nil { - return UnknownSource, fmt.Errorf("unable to seek archive=%s: %w", imgPath, err) - } - - var fileErr *file.ErrFileNotFound - _, err = file.ReaderFromTar(f, pair.path) - if err == nil { - return pair.source, nil - } else if !errors.As(err, &fileErr) { - // short-circuit, there is something wrong with the tar reading process - return UnknownSource, err - } - } - - // there are no other archive-based formats supported - return UnknownSource, nil -} - -// String returns a convenient display string for the source. -func (t Source) String() string { - return sourceStr[t] -} diff --git a/pkg/image/source_test.go b/pkg/image/source_test.go deleted file mode 100644 index e05eb10b..00000000 --- a/pkg/image/source_test.go +++ /dev/null @@ -1,478 +0,0 @@ -package image - -import ( - "archive/tar" - "io" - "os" - "path" - "strings" - "testing" - - "github.com/mitchellh/go-homedir" - "github.com/spf13/afero" - "github.com/sylabs/sif/v2/pkg/sif" -) - -func TestDetectSource(t *testing.T) { - cases := []struct { - name string - fs afero.Fs - input string - source Source - expectedLocation string - }{ - { - name: "podman-engine", - input: "podman:something:latest", - source: PodmanDaemonSource, - expectedLocation: "something:latest", - }, - { - name: "docker-archive", - input: "docker-archive:a/place.tar", - source: DockerTarballSource, - expectedLocation: "a/place.tar", - }, - { - name: "docker-engine-by-possible-id", - input: "a5e", - source: UnknownSource, - expectedLocation: "", - }, - { - name: "docker-engine-impossible-id", - // not a valid ID - input: "a5E", - source: UnknownSource, - expectedLocation: "", - }, - { - name: "docker-engine", - input: "docker:something/something:latest", - source: DockerDaemonSource, - expectedLocation: "something/something:latest", - }, - { - name: "docker-engine-edge-case", - input: "docker:latest", - source: DockerDaemonSource, - // we want to be able to handle this case better, however, I don't see a way to do this - // the user will need to provide more explicit input (docker:docker:latest) - expectedLocation: "latest", - }, - { - name: "docker-engine-edge-case-explicit", - input: "docker:docker:latest", - source: DockerDaemonSource, - expectedLocation: "docker:latest", - }, - { - name: "docker-caps", - input: "DoCKEr:something/something:latest", - source: DockerDaemonSource, - expectedLocation: "something/something:latest", - }, - { - name: "infer-docker-engine", - input: "something/something:latest", - source: UnknownSource, - expectedLocation: "", - }, - { - name: "bad-hint", - input: "blerg:something/something:latest", - source: UnknownSource, - expectedLocation: "", - }, - { - name: "relative-path-1", - input: ".", - source: UnknownSource, - expectedLocation: "", - }, - { - name: "relative-path-2", - input: "./", - source: UnknownSource, - expectedLocation: "", - }, - { - name: "relative-parent-path", - input: "../", - source: UnknownSource, - expectedLocation: "", - }, - { - name: "oci-tar-path", - fs: getDummyTar(t, "a-potential/path", "oci-layout"), - input: "a-potential/path", - source: OciTarballSource, - expectedLocation: "a-potential/path", - }, - { - name: "unparsable-existing-path", - fs: getDummyTar(t, "a-potential/path"), - input: "a-potential/path", - source: UnknownSource, - expectedLocation: "", - }, - // honor tilde expansion - { - name: "oci-tar-path", - fs: getDummyTar(t, "~/a-potential/path", "oci-layout"), - input: "~/a-potential/path", - source: OciTarballSource, - expectedLocation: "~/a-potential/path", - }, - { - name: "oci-tar-path-explicit", - fs: getDummyTar(t, "~/a-potential/path", "oci-layout"), - input: "oci-archive:~/a-potential/path", - source: OciTarballSource, - expectedLocation: "~/a-potential/path", - }, - { - name: "oci-tar-path-with-scheme-separator", - fs: getDummyTar(t, "a-potential/path:version", "oci-layout"), - input: "a-potential/path:version", - source: OciTarballSource, - expectedLocation: "a-potential/path:version", - }, - { - name: "singularity-path", - fs: getDummySIF(t, "~/a-potential/path.sif"), - input: "singularity:~/a-potential/path.sif", - source: SingularitySource, - expectedLocation: "~/a-potential/path.sif", - }, - { - name: "singularity-path-tilde", - fs: getDummySIF(t, "~/a-potential/path.sif"), - input: "~/a-potential/path.sif", - source: SingularitySource, - expectedLocation: "~/a-potential/path.sif", - }, - { - name: "singularity-path-explicit", - fs: getDummySIF(t, "~/a-potential/path.sif"), - input: "singularity:~/a-potential/path.sif", - source: SingularitySource, - expectedLocation: "~/a-potential/path.sif", - }, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - fs := c.fs - if c.fs == nil { - fs = afero.NewMemMapFs() - } - - source, location, err := detectSource(fs, c.input) - if err != nil { - t.Fatalf("unexecpted error: %+v", err) - } - if c.source != source { - t.Errorf("expected: %q , got: %q", c.source, source) - } - - // lean on the users real home directory value - expandedExpectedLocation, err := homedir.Expand(c.expectedLocation) - if err != nil { - t.Fatalf("unable to expand path=%q: %+v", c.expectedLocation, err) - } - - if expandedExpectedLocation != location { - t.Errorf("expected: %q , got: %q", expandedExpectedLocation, location) - } - }) - } -} - -func TestParseScheme(t *testing.T) { - cases := []struct { - source string - expected Source - }{ - { - // regression for unsupported behavior - source: "tar", - expected: UnknownSource, - }, - { - // regression for unsupported behavior - source: "tarball", - expected: UnknownSource, - }, - { - // regression for unsupported behavior - source: "archive", - expected: UnknownSource, - }, - { - source: "docker-archive", - expected: DockerTarballSource, - }, - { - // regression for unsupported behavior - source: "docker-tar", - expected: UnknownSource, - }, - { - // regression for unsupported behavior - source: "docker-tarball", - expected: UnknownSource, - }, - { - source: "Docker", - expected: DockerDaemonSource, - }, - { - source: "DOCKER", - expected: DockerDaemonSource, - }, - { - source: "docker", - expected: DockerDaemonSource, - }, - { - // regression for unsupported behavior - source: "docker-daemon", - expected: UnknownSource, - }, - { - // regression for unsupported behavior - source: "docker-engine", - expected: UnknownSource, - }, - { - source: "oci-archive", - expected: OciTarballSource, - }, - { - // regression for unsupported behavior - source: "oci-tar", - expected: UnknownSource, - }, - { - // regression for unsupported behavior - source: "oci-tarball", - expected: UnknownSource, - }, - { - // regression for unsupported behavior - source: "oci", - expected: UnknownSource, - }, - { - source: "oci-dir", - expected: OciDirectorySource, - }, - { - // regression for unsupported behavior - source: "oci-directory", - expected: UnknownSource, - }, - { - source: "", - expected: UnknownSource, - }, - { - source: "something", - expected: UnknownSource, - }, - } - for _, c := range cases { - actual := ParseSourceScheme(c.source) - if c.expected != actual { - t.Errorf("unexpected source: %s!=%s", c.expected, actual) - } - } -} - -func TestDetectSourceFromPath(t *testing.T) { - tests := []struct { - name string - path string - fs afero.Fs - expectedSource Source - expectedErr bool - }{ - { - name: "no tar paths", - path: "image.tar", - fs: getDummyTar(t, "image.tar"), - expectedSource: UnknownSource, - }, - { - name: "dummy tar paths", - path: "image.tar", - fs: getDummyTar(t, "image.tar", "manifest", "index", "oci_layout"), - expectedSource: UnknownSource, - }, - { - name: "oci-layout tar path", - path: "image.tar", - fs: getDummyTar(t, "image.tar", "oci-layout"), - expectedSource: OciTarballSource, - }, - { - name: "index.json tar path", - path: "image.tar", - fs: getDummyTar(t, "image.tar", "index.json"), // this is an optional OCI file... - expectedSource: UnknownSource, // ...which we should not respond to as primary evidence - }, - { - name: "docker tar path", - path: "image.tar", - fs: getDummyTar(t, "image.tar", "manifest.json"), - expectedSource: DockerTarballSource, - }, - { - name: "no dir paths", - path: "image", - fs: getDummyDir(t, "image"), - expectedSource: UnknownSource, - }, - { - name: "oci-layout path", - path: "image", - fs: getDummyDir(t, "image", "oci-layout"), - expectedSource: OciDirectorySource, - }, - { - name: "dummy dir paths", - path: "image", - fs: getDummyDir(t, "image", "manifest", "index", "oci_layout"), - expectedSource: UnknownSource, - }, - { - name: "no path given", - path: "/does-not-exist", - expectedSource: UnknownSource, - expectedErr: false, - }, - { - name: "singularity path", - path: "image.sif", - fs: getDummySIF(t, "image.sif"), - expectedSource: SingularitySource, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fs := test.fs - if fs == nil { - fs = afero.NewMemMapFs() - } - - actual, err := detectSourceFromPath(fs, test.path) - if err != nil && !test.expectedErr { - t.Fatalf("unexpected error: %+v", err) - } else if err == nil && test.expectedErr { - t.Fatal("expected error but got none") - } - if actual != test.expectedSource { - t.Errorf("unexpected source: %+v (expected: %+v)", actual, test.expectedSource) - } - }) - } -} - -// getDummyTar returns a filesystem that contains a TAR archive at archivePath populated with paths. -func getDummyTar(t *testing.T, archivePath string, paths ...string) afero.Fs { - t.Helper() - - archivePath, err := homedir.Expand(archivePath) - if err != nil { - t.Fatalf("unable to expand home path=%q: %+v", archivePath, err) - } - - fs := afero.NewMemMapFs() - - testFile, err := fs.Create(archivePath) - if err != nil { - t.Fatalf("failed to create dummy tar: %+v", err) - } - - tarWriter := tar.NewWriter(testFile) - defer tarWriter.Close() - - for _, filePath := range paths { - header := &tar.Header{ - Name: filePath, - Size: 13, - } - - err = tarWriter.WriteHeader(header) - if err != nil { - t.Fatalf("could not write dummy header: %+v", err) - } - - _, err = io.Copy(tarWriter, strings.NewReader("hello, world!")) - if err != nil { - t.Fatalf("could not write dummy file: %+v", err) - } - } - - return fs -} - -// getDummyDir returns a filesystem that contains directory dirPath populated with paths. -func getDummyDir(t *testing.T, dirPath string, paths ...string) afero.Fs { - t.Helper() - - dirPath, err := homedir.Expand(dirPath) - if err != nil { - t.Fatalf("unable to expand home dir=%q: %+v", dirPath, err) - } - - fs := afero.NewMemMapFs() - - if err = fs.Mkdir(dirPath, os.ModePerm); err != nil { - t.Fatalf("failed to create dummy tar: %+v", err) - } - - for _, filePath := range paths { - f, err := fs.Create(path.Join(dirPath, filePath)) - if err != nil { - t.Fatalf("unable to create file: %+v", err) - } - - if _, err = f.WriteString("hello, world!"); err != nil { - t.Fatalf("unable to write file") - } - - if err = f.Close(); err != nil { - t.Fatalf("unable to close file") - } - } - - return fs -} - -// getDummySIF returns a filesystem that contains a SIF at path. -func getDummySIF(t *testing.T, path string, opts ...sif.CreateOpt) afero.Fs { - t.Helper() - - path, err := homedir.Expand(path) - if err != nil { - t.Fatalf("unable to expand home dir=%q: %+v", path, err) - } - - fs := afero.NewMemMapFs() - - f, err := fs.Create(path) - if err != nil { - t.Fatalf("failed to create file: %+v", err) - } - defer f.Close() - - fi, err := sif.CreateContainer(f, opts...) - if err != nil { - t.Fatalf("failed to create container: %+v", err) - } - defer fi.UnloadContainer() - - return fs -} diff --git a/pkg/imagetest/image_fixtures.go b/pkg/imagetest/image_fixtures.go index 1c2b2165..2dfaacea 100644 --- a/pkg/imagetest/image_fixtures.go +++ b/pkg/imagetest/image_fixtures.go @@ -27,10 +27,8 @@ const ( func PrepareFixtureImage(t testing.TB, source, name string) string { t.Helper() - sourceObj := image.ParseSourceScheme(source) - var location string - switch sourceObj { + switch source { case image.ContainerdDaemonSource: location = LoadFixtureImageIntoContainerd(t, name) case image.DockerTarballSource: diff --git a/providers.go b/providers.go new file mode 100644 index 00000000..fcd19f56 --- /dev/null +++ b/providers.go @@ -0,0 +1,54 @@ +package stereoscope + +import ( + "github.com/anchore/go-collections" + containerdClient "github.com/anchore/stereoscope/internal/containerd" + "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/stereoscope/pkg/image/containerd" + "github.com/anchore/stereoscope/pkg/image/docker" + "github.com/anchore/stereoscope/pkg/image/oci" + "github.com/anchore/stereoscope/pkg/image/podman" + "github.com/anchore/stereoscope/pkg/image/sif" +) + +const ( + FileTag = "file" + DirTag = "dir" + DaemonTag = "daemon" + PullTag = "pull" + RegistryTag = "registry" +) + +// ImageProviderConfig is the uber-configuration containing all configuration needed by stereoscope image providers +type ImageProviderConfig struct { + UserInput string + Platform *image.Platform + Registry image.RegistryOptions +} + +func ImageProviders(cfg ImageProviderConfig) []collections.TaggedValue[image.Provider] { + tempDirGenerator := rootTempDirGenerator.NewGenerator() + return []collections.TaggedValue[image.Provider]{ + // file providers + taggedProvider(docker.NewArchiveProvider(tempDirGenerator, cfg.UserInput), FileTag), + taggedProvider(oci.NewArchiveProvider(tempDirGenerator, cfg.UserInput), FileTag), + taggedProvider(oci.NewDirectoryProvider(tempDirGenerator, cfg.UserInput), FileTag, DirTag), + taggedProvider(sif.NewArchiveProvider(tempDirGenerator, cfg.UserInput), FileTag), + + // daemon providers + taggedProvider(docker.NewDaemonProvider(tempDirGenerator, cfg.UserInput, cfg.Platform), DaemonTag, PullTag), + taggedProvider(podman.NewDaemonProvider(tempDirGenerator, cfg.UserInput, cfg.Platform), DaemonTag, PullTag), + taggedProvider(containerd.NewDaemonProvider(tempDirGenerator, cfg.Registry, containerdClient.Namespace(), cfg.UserInput, cfg.Platform), DaemonTag, PullTag), + + // registry providers + taggedProvider(oci.NewRegistryProvider(tempDirGenerator, cfg.Registry, cfg.UserInput, cfg.Platform), RegistryTag, PullTag), + } +} + +func taggedProvider(provider image.Provider, tags ...string) collections.TaggedValue[image.Provider] { + return collections.NewTaggedValue[image.Provider](provider, append([]string{provider.Name()}, tags...)...) +} + +func allProviderTags() []string { + return collections.TaggedValueSet[image.Provider]{}.Join(ImageProviders(ImageProviderConfig{})...).Tags() +} diff --git a/test/integration/fixture_image_simple_test.go b/test/integration/fixture_image_simple_test.go index 1360a715..ff167cb2 100644 --- a/test/integration/fixture_image_simple_test.go +++ b/test/integration/fixture_image_simple_test.go @@ -12,9 +12,9 @@ import ( "testing" v1Types "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/scylladb/go-set" "github.com/stretchr/testify/require" + "github.com/anchore/go-collections" "github.com/anchore/stereoscope" "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree" @@ -120,11 +120,9 @@ type testCase struct { } func TestSimpleImage(t *testing.T) { - expectedSet := set.NewIntSet() - for _, src := range image.AllSources { - expectedSet.Add(int(src)) - } - expectedSet.Remove(int(image.OciRegistrySource)) + expectedSet := collections.TaggedValueSet[image.Provider]{}. + Join(stereoscope.ImageProviders(stereoscope.ImageProviderConfig{})...). + Remove(image.OciRegistrySource) for _, c := range simpleImageTestCases { t.Run(c.source, func(t *testing.T) { @@ -149,7 +147,7 @@ func TestSimpleImage(t *testing.T) { }) } - if len(simpleImageTestCases) < expectedSet.Size() { + if len(simpleImageTestCases) < len(expectedSet) { t.Fatalf("probably missed a source during testing, double check that all image.sources are covered") } diff --git a/test/integration/fixture_image_symlinks_test.go b/test/integration/fixture_image_symlinks_test.go index 63a65e1d..cd1f5850 100644 --- a/test/integration/fixture_image_symlinks_test.go +++ b/test/integration/fixture_image_symlinks_test.go @@ -9,9 +9,10 @@ import ( "runtime" "testing" - "github.com/scylladb/go-set" "github.com/stretchr/testify/require" + "github.com/anchore/go-collections" + "github.com/anchore/stereoscope" "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree" "github.com/anchore/stereoscope/pkg/image" @@ -63,11 +64,9 @@ func TestImageSymlinks(t *testing.T) { }, } - expectedSet := set.NewIntSet() - for _, src := range image.AllSources { - expectedSet.Add(int(src)) - } - expectedSet.Remove(int(image.OciRegistrySource)) + expectedSet := collections.TaggedValueSet[image.Provider]{}. + Join(stereoscope.ImageProviders(stereoscope.ImageProviderConfig{})...). + Remove(image.OciRegistrySource) for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -91,7 +90,7 @@ func TestImageSymlinks(t *testing.T) { }) } - if len(cases) < expectedSet.Size() { + if len(cases) < len(expectedSet) { t.Fatalf("probably missed a source during testing, double check that all image.sources are covered") } diff --git a/test/integration/platform_test.go b/test/integration/platform_test.go index 8a01addd..0de32999 100644 --- a/test/integration/platform_test.go +++ b/test/integration/platform_test.go @@ -92,7 +92,7 @@ func TestPlatformSelection(t *testing.T) { for _, tt := range tests { platform := fmt.Sprintf("%s/%s", tt.os, tt.architecture) - t.Run(fmt.Sprintf("%s/%s", tt.source.String(), platform), func(t *testing.T) { + t.Run(fmt.Sprintf("%s/%s", tt.source, platform), func(t *testing.T) { if runtime.GOOS != "linux" { switch tt.source { case image.ContainerdDaemonSource: