From fafb1ab308c7e0488eff5dbfcd4996411ba6f060 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 25 Jan 2024 17:20:13 +0100 Subject: [PATCH] [8.12](backport #4139) package_test.go - Support OCI Image Layout (#4141) * package_test.go - Support OCI Image Layout (#4139) * fixed package_test for newer docker format * Fixed docker packaging * remove unwanted error package from copy.go (cherry picked from commit ca5fdacb66acb1e42bab130fe1af4669cfa2a552) # Conflicts: # dev-tools/packaging/package_test.go * slice contains fixed --------- Co-authored-by: Michal Pristas --- dev-tools/mage/copy.go | 26 +++---- dev-tools/packaging/package_test.go | 111 ++++++++++++++++++++-------- 2 files changed, 92 insertions(+), 45 deletions(-) diff --git a/dev-tools/mage/copy.go b/dev-tools/mage/copy.go index f3daef703f3..c64d5cc6561 100644 --- a/dev-tools/mage/copy.go +++ b/dev-tools/mage/copy.go @@ -11,8 +11,6 @@ import ( "os" "path/filepath" "regexp" - - "github.com/pkg/errors" ) // Copy copies a file or a directory (recursively) and preserves the permissions. @@ -34,22 +32,25 @@ type CopyTask struct { // Execute executes the copy and returns an error of there is a failure. func (t *CopyTask) Execute() error { if err := t.init(); err != nil { - return errors.Wrap(err, "copy failed") + return fmt.Errorf("copy failed: %w", err) } info, err := os.Stat(t.Source) if err != nil { - return errors.Wrapf(err, "copy failed: cannot stat source file %v", t.Source) + return fmt.Errorf("copy failed: cannot stat source file %v: %w", t.Source, err) } - return errors.Wrap(t.recursiveCopy(t.Source, t.Dest, fs.FileInfoToDirEntry(info)), "copy failed") + if err := t.recursiveCopy(t.Source, t.Dest, fs.FileInfoToDirEntry(info)); err != nil { + return fmt.Errorf("copy failed: %w", err) + } + return nil } func (t *CopyTask) init() error { for _, excl := range t.Exclude { re, err := regexp.Compile(excl) if err != nil { - return errors.Wrapf(err, "bad exclude pattern %v", excl) + return fmt.Errorf("bad exclude pattern %v: %w", excl, err) } t.excludes = append(t.excludes, re) } @@ -89,8 +90,7 @@ func (t *CopyTask) fileCopy(src, dest string, entry fs.DirEntry) error { } if !info.Mode().IsRegular() { - return errors.Errorf("failed to copy source file because it is not a " + - "regular file") + return fmt.Errorf("failed to copy source file because it is not a regular file") } mode := t.Mode @@ -126,19 +126,19 @@ func (t *CopyTask) dirCopy(src, dest string, entry fs.DirEntry) error { } if err := os.MkdirAll(dest, mode&os.ModePerm); err != nil { - return errors.Wrap(err, "failed creating dirs") + return fmt.Errorf("failed creating dirs: %w", err) } contents, err := os.ReadDir(src) if err != nil { - return errors.Wrapf(err, "failed to read dir %v", src) + return fmt.Errorf("failed to read dir %v: %w", src, err) } for _, entry := range contents { - srcFile := filepath.Join(src, info.Name()) - destFile := filepath.Join(dest, info.Name()) + srcFile := filepath.Join(src, entry.Name()) + destFile := filepath.Join(dest, entry.Name()) if err = t.recursiveCopy(srcFile, destFile, entry); err != nil { - return errors.Wrapf(err, "failed to copy %v to %v", srcFile, destFile) + return fmt.Errorf("failed to copy %v to %v: %w", srcFile, destFile, err) } } diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index 86c8f760a9d..10ba122ed06 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -94,6 +94,7 @@ func TestZip(t *testing.T) { func TestDocker(t *testing.T) { dockers := getFiles(t, regexp.MustCompile(`\.docker\.tar\.gz$`)) for _, docker := range dockers { + t.Log(docker) checkDocker(t, docker) } } @@ -699,13 +700,19 @@ func readZip(t *testing.T, zipFile string, inspectors ...inspector) (*packageFil } func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { + // Read the manifest file first so that the config file and layer + // names are known in advance. + manifest, err := getDockerManifest(dockerFile) + if err != nil { + return nil, nil, err + } + file, err := os.Open(dockerFile) if err != nil { return nil, nil, err } defer file.Close() - var manifest *dockerManifest var info *dockerInfo layers := make(map[string]*packageFile) @@ -716,7 +723,6 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { defer gzipReader.Close() tarReader := tar.NewReader(gzipReader) - manifestFileName := "manifest.json" for { header, err := tarReader.Next() if err != nil { @@ -727,22 +733,17 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { } switch { - case header.Name == manifestFileName: - manifest, err = readDockerManifest(tarReader) - if err != nil { - return nil, nil, err - } - case strings.HasSuffix(header.Name, ".json") && header.Name != manifestFileName: + case header.Name == manifest.Config: info, err = readDockerInfo(tarReader) if err != nil { return nil, nil, err } - case strings.HasSuffix(header.Name, "/layer.tar"): + case sliceContains(manifest.Layers, header.Name): layer, err := readTarContents(header.Name, tarReader) if err != nil { return nil, nil, err } - layers[filepath.Dir(header.Name)] = layer + layers[header.Name] = layer } } @@ -756,10 +757,9 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { // Read layers in order and for each file keep only the entry seen in the later layer p := &packageFile{Name: filepath.Base(dockerFile), Contents: map[string]packageEntry{}} for _, layer := range manifest.Layers { - layerID := filepath.Dir(layer) - layerFile, found := layers[layerID] + layerFile, found := layers[layer] if !found { - return nil, nil, fmt.Errorf("layer not found: %s", layerID) + return nil, nil, fmt.Errorf("layer not found: %s", layer) } for name, entry := range layerFile.Contents { if excludedPathsPattern.MatchString(name) { @@ -791,25 +791,6 @@ type dockerManifest struct { Layers []string } -func readDockerManifest(r io.Reader) (*dockerManifest, error) { - data, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - - var manifests []*dockerManifest - err = json.Unmarshal(data, &manifests) - if err != nil { - return nil, err - } - - if len(manifests) != 1 { - return nil, fmt.Errorf("one and only one manifest expected, %d found", len(manifests)) - } - - return manifests[0], nil -} - type dockerInfo struct { Config struct { Entrypoint []string @@ -833,3 +814,69 @@ func readDockerInfo(r io.Reader) (*dockerInfo, error) { return &info, nil } + +// getDockerManifest opens a gzipped tar file to read the Docker manifest.json +// that it is expected to contain. +func getDockerManifest(file string) (*dockerManifest, error) { + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + + gzipReader, err := gzip.NewReader(f) + if err != nil { + return nil, err + } + defer gzipReader.Close() + + var manifest *dockerManifest + tarReader := tar.NewReader(gzipReader) + for { + header, err := tarReader.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + + if header.Name == "manifest.json" { + manifest, err = readDockerManifest(tarReader) + if err != nil { + return nil, err + } + break + } + } + + return manifest, nil +} + +func readDockerManifest(r io.Reader) (*dockerManifest, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + var manifests []*dockerManifest + err = json.Unmarshal(data, &manifests) + if err != nil { + return nil, err + } + + if len(manifests) != 1 { + return nil, fmt.Errorf("one and only one manifest expected, %d found", len(manifests)) + } + + return manifests[0], nil +} + +func sliceContains(s []string, e string) bool { + for _, v := range s { + if e == v { + return true + } + } + return false +}