From ca962cecfd470ed3861e3965220b4fb6213aced5 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 8 Jan 2025 17:22:01 +0100 Subject: [PATCH 1/3] image/tarexport: add fork of os.MkdirAll (non-buildable commit) Fork the os.MkdirAll function, so that we can make changes to add support for setting atime and mtime. This commit contains a 1:1 fork, but won't build; code is taken from: https://cs.opensource.google/go/go/+/refs/tags/go1.23.4:src/os/path.go;l=19-66 Signed-off-by: Sebastiaan van Stijn --- image/tarexport/os_path.go | 61 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 image/tarexport/os_path.go diff --git a/image/tarexport/os_path.go b/image/tarexport/os_path.go new file mode 100644 index 0000000000000..08ba0f0c0cc07 --- /dev/null +++ b/image/tarexport/os_path.go @@ -0,0 +1,61 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code in this file is a modified version of go stdlib; +// https://cs.opensource.google/go/go/+/refs/tags/go1.23.4:src/os/path.go;l=19-66 + +package tarexport + +import ( + "syscall" +) + +func MkdirAll(path string, perm FileMode) error { + // Fast path: if we can tell whether path is a directory or file, stop with success or error. + dir, err := Stat(path) + if err == nil { + if dir.IsDir() { + return nil + } + return &PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR} + } + + // Slow path: make sure parent exists and then call Mkdir for path. + + // Extract the parent folder from path by first removing any trailing + // path separator and then scanning backward until finding a path + // separator or reaching the beginning of the string. + i := len(path) - 1 + for i >= 0 && IsPathSeparator(path[i]) { + i-- + } + for i >= 0 && !IsPathSeparator(path[i]) { + i-- + } + if i < 0 { + i = 0 + } + + // If there is a parent directory, and it is not the volume name, + // recurse to ensure parent directory exists. + if parent := path[:i]; len(parent) > len(filepathlite.VolumeName(path)) { + err = MkdirAll(parent, perm) + if err != nil { + return err + } + } + + // Parent now exists; invoke Mkdir and use its result. + err = Mkdir(path, perm) + if err != nil { + // Handle arguments like "foo/." by + // double-checking that directory doesn't exist. + dir, err1 := Lstat(path) + if err1 == nil && dir.IsDir() { + return nil + } + return err + } + return nil +} From e324df3f1b5b91f8edc33a27d571ebdd7c06cb8c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 8 Jan 2025 17:48:29 +0100 Subject: [PATCH 2/3] image/tarexport: patch MkDirall to accept atime, mtime Signed-off-by: Sebastiaan van Stijn --- image/tarexport/os_path.go | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/image/tarexport/os_path.go b/image/tarexport/os_path.go index 08ba0f0c0cc07..4b289034a4a8a 100644 --- a/image/tarexport/os_path.go +++ b/image/tarexport/os_path.go @@ -8,17 +8,26 @@ package tarexport import ( + "fmt" + "os" + "path/filepath" "syscall" + "time" + + "github.com/docker/docker/pkg/system" ) -func MkdirAll(path string, perm FileMode) error { +// mkdirAllWithChtimes is nearly an identical copy to the [os.MkdirAll] but +// tracks created directories and applies the provided mtime and atime using +// [system.Chtimes]. +func mkdirAllWithChtimes(path string, perm os.FileMode, atime, mtime time.Time) error { // Fast path: if we can tell whether path is a directory or file, stop with success or error. - dir, err := Stat(path) + dir, err := os.Stat(path) if err == nil { if dir.IsDir() { return nil } - return &PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR} + return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR} } // Slow path: make sure parent exists and then call Mkdir for path. @@ -27,10 +36,10 @@ func MkdirAll(path string, perm FileMode) error { // path separator and then scanning backward until finding a path // separator or reaching the beginning of the string. i := len(path) - 1 - for i >= 0 && IsPathSeparator(path[i]) { + for i >= 0 && os.IsPathSeparator(path[i]) { i-- } - for i >= 0 && !IsPathSeparator(path[i]) { + for i >= 0 && !os.IsPathSeparator(path[i]) { i-- } if i < 0 { @@ -39,23 +48,27 @@ func MkdirAll(path string, perm FileMode) error { // If there is a parent directory, and it is not the volume name, // recurse to ensure parent directory exists. - if parent := path[:i]; len(parent) > len(filepathlite.VolumeName(path)) { - err = MkdirAll(parent, perm) + if parent := path[:i]; len(parent) > len(filepath.VolumeName(path)) { + err = mkdirAllWithChtimes(parent, perm, atime, mtime) if err != nil { return err } } // Parent now exists; invoke Mkdir and use its result. - err = Mkdir(path, perm) + err = os.Mkdir(path, perm) if err != nil { // Handle arguments like "foo/." by // double-checking that directory doesn't exist. - dir, err1 := Lstat(path) + dir, err1 := os.Lstat(path) if err1 == nil && dir.IsDir() { return nil } return err } + + if err := system.Chtimes(path, atime, mtime); err != nil { + return fmt.Errorf("applying atime=%v and mtime=%v: %w", atime, mtime, err) + } return nil } From 392d33c98c8c8ec956d60657752595023986a310 Mon Sep 17 00:00:00 2001 From: Stephen Day Date: Wed, 9 Oct 2024 08:02:29 -0700 Subject: [PATCH 3/3] image/save: set a stable timestamp for assets When saving a docker image with `docker save`, output may have the current timestamp, resulting in slightly changed content each time the `save` command gets run. This patch attemtps to stabilize that effort to clean up some spots where we've missed setting the timestamps. It's not totally clear that setting these timestamps to 0 is the correct behavior but it will fix the hash stability problem on output. Signed-off-by: Stephen Day Signed-off-by: Sebastiaan van Stijn --- image/tarexport/save.go | 62 ++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/image/tarexport/save.go b/image/tarexport/save.go index 49f3b871ed107..78bfdbaffc274 100644 --- a/image/tarexport/save.go +++ b/image/tarexport/save.go @@ -261,7 +261,7 @@ func (s *saveSession) save(ctx context.Context, outStream io.Writer) error { dgst := digest.FromBytes(data) mFile := filepath.Join(s.outDir, ocispec.ImageBlobsDir, dgst.Algorithm().String(), dgst.Encoded()) - if err := os.MkdirAll(filepath.Dir(mFile), 0o755); err != nil { + if err := mkdirAllWithChtimes(filepath.Dir(mFile), 0o755, time.Unix(0, 0), time.Unix(0, 0)); err != nil { return errors.Wrap(err, "error creating blob directory") } if err := system.Chtimes(filepath.Dir(mFile), time.Unix(0, 0), time.Unix(0, 0)); err != nil { @@ -385,6 +385,9 @@ func (s *saveSession) save(ctx context.Context, outStream io.Writer) error { if err := os.WriteFile(idxFile, data, 0o644); err != nil { return errors.Wrap(err, "error writing oci index file") } + if err := system.Chtimes(idxFile, time.Unix(0, 0), time.Unix(0, 0)); err != nil { + return errors.Wrap(err, "error setting oci index file timestamps") + } return s.writeTar(ctx, tempDir, outStream) } @@ -419,6 +422,11 @@ func (s *saveSession) saveImage(ctx context.Context, id image.ID) (_ map[layer.D return nil, fmt.Errorf("empty export - not implemented") } + ts := time.Unix(0, 0) + if img.Created != nil { + ts = *img.Created + } + var parent digest.Digest var layers []layer.DiffID var foreignSrcs map[layer.DiffID]distribution.Descriptor @@ -450,7 +458,7 @@ func (s *saveSession) saveImage(ctx context.Context, id image.ID) (_ map[layer.D } v1Img.OS = img.OS - src, err := s.saveConfigAndLayer(ctx, rootFS.ChainID(), v1Img, img.Created) + src, err := s.saveConfigAndLayer(ctx, rootFS.ChainID(), v1Img, &ts) if err != nil { return nil, err } @@ -469,26 +477,22 @@ func (s *saveSession) saveImage(ctx context.Context, id image.ID) (_ map[layer.D dgst := digest.FromBytes(data) blobDir := filepath.Join(s.outDir, ocispec.ImageBlobsDir, dgst.Algorithm().String()) - if err := os.MkdirAll(blobDir, 0o755); err != nil { + if err := mkdirAllWithChtimes(blobDir, 0o755, ts, ts); err != nil { return nil, err } - if img.Created != nil { - if err := system.Chtimes(blobDir, *img.Created, *img.Created); err != nil { - return nil, err - } - if err := system.Chtimes(filepath.Dir(blobDir), *img.Created, *img.Created); err != nil { - return nil, err - } + if err := system.Chtimes(blobDir, ts, ts); err != nil { + return nil, err + } + if err := system.Chtimes(filepath.Dir(blobDir), ts, ts); err != nil { + return nil, err } configFile := filepath.Join(blobDir, dgst.Encoded()) if err := os.WriteFile(configFile, img.RawJSON(), 0o644); err != nil { return nil, err } - if img.Created != nil { - if err := system.Chtimes(configFile, *img.Created, *img.Created); err != nil { - return nil, err - } + if err := system.Chtimes(configFile, ts, ts); err != nil { + return nil, err } s.images[id].layers = layers @@ -506,6 +510,11 @@ func (s *saveSession) saveConfigAndLayer(ctx context.Context, id layer.ChainID, span.SetStatus(outErr) }() + ts := time.Unix(0, 0) + if createdTime != nil { + ts = *createdTime + } + outDir := filepath.Join(s.outDir, ocispec.ImageBlobsDir) if _, ok := s.savedConfigs[legacyImg.ID]; !ok { @@ -542,7 +551,7 @@ func (s *saveSession) saveConfigAndLayer(ctx context.Context, id layer.ChainID, // We use sequential file access to avoid depleting the standby list on // Windows. On Linux, this equates to a regular os.Create. - if err := os.MkdirAll(filepath.Dir(layerPath), 0o755); err != nil { + if err := mkdirAllWithChtimes(filepath.Dir(layerPath), 0o755, ts, ts); err != nil { return distribution.Descriptor{}, errors.Wrap(err, "could not create layer dir parent") } tarFile, err := sequential.Create(layerPath) @@ -576,12 +585,10 @@ func (s *saveSession) saveConfigAndLayer(ctx context.Context, id layer.ChainID, layerPath = filepath.Join(outDir, lDgst.Algorithm().String(), lDgst.Encoded()) } - if createdTime != nil { - for _, fname := range []string{outDir, layerPath} { - // todo: maybe save layer created timestamp? - if err := system.Chtimes(fname, *createdTime, *createdTime); err != nil { - return distribution.Descriptor{}, errors.Wrap(err, "could not set layer timestamp") - } + for _, fname := range []string{outDir, layerPath} { + // todo: maybe save layer created timestamp? + if err := system.Chtimes(fname, ts, ts); err != nil { + return distribution.Descriptor{}, errors.Wrap(err, "could not set layer timestamp") } } @@ -608,9 +615,14 @@ func (s *saveSession) saveConfig(legacyImg image.V1Image, outDir string, created return err } + ts := time.Unix(0, 0) + if createdTime != nil { + ts = *createdTime + } + cfgDgst := digest.FromBytes(imageConfig) configPath := filepath.Join(outDir, cfgDgst.Algorithm().String(), cfgDgst.Encoded()) - if err := os.MkdirAll(filepath.Dir(configPath), 0o755); err != nil { + if err := mkdirAllWithChtimes(filepath.Dir(configPath), 0o755, ts, ts); err != nil { return errors.Wrap(err, "could not create layer dir parent") } @@ -618,10 +630,8 @@ func (s *saveSession) saveConfig(legacyImg image.V1Image, outDir string, created return err } - if createdTime != nil { - if err := system.Chtimes(configPath, *createdTime, *createdTime); err != nil { - return errors.Wrap(err, "could not set config timestamp") - } + if err := system.Chtimes(configPath, ts, ts); err != nil { + return errors.Wrap(err, "could not set config timestamp") } s.savedConfigs[legacyImg.ID] = struct{}{}