Merge pull request #5647 from nalind/linked-layers

Add PrependedLinkedLayers/AppendedLinkedLayers to CommitOptions

buildah.go

@@ -191,6 +191,13 @@ type Builder struct {
 	// CDIConfigDir is the location of CDI configuration files, if the files in
 	// the default configuration locations shouldn't be used.
 	CDIConfigDir string
+	// PrependedLinkedLayers and AppendedLinkedLayers are combinations of
+	// history entries and locations of either directory trees (if
+	// directories, per os.Stat()) or uncompressed layer blobs which should
+	// be added to the image at commit-time.  The order of these relative
+	// to PrependedEmptyLayers and AppendedEmptyLayers in the committed
+	// image is not guaranteed.
+	PrependedLinkedLayers, AppendedLinkedLayers []LinkedLayer
 }
 
 // BuilderInfo are used as objects to display container information

commit.go

@@ -24,6 +24,7 @@ import (
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/stringid"
 	digest "github.com/opencontainers/go-digest"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/exp/maps"
 )
@@ -120,10 +121,11 @@ type CommitOptions struct {
 	// OverrideConfig is applied.
 	OverrideChanges []string
 	// ExtraImageContent is a map which describes additional content to add
-	// to the committed image.  The map's keys are filesystem paths in the
-	// image and the corresponding values are the paths of files whose
-	// contents will be used in their place.  The contents will be owned by
-	// 0:0 and have mode 0644.  Currently only accepts regular files.
+	// to the new layer in the committed image.  The map's keys are
+	// filesystem paths in the image and the corresponding values are the
+	// paths of files whose contents will be used in their place.  The
+	// contents will be owned by 0:0 and have mode 0o644.  Currently only
+	// accepts regular files.
 	ExtraImageContent map[string]string
 	// SBOMScanOptions encapsulates options which control whether or not we
 	// run scanners on the rootfs that we're about to commit, and how.
@@ -132,6 +134,23 @@ type CommitOptions struct {
 	// the image in Docker format.  Newer BuildKit-based builds don't set
 	// this field.
 	CompatSetParent types.OptionalBool
+	// PrependedLinkedLayers and AppendedLinkedLayers are combinations of
+	// history entries and locations of either directory trees (if
+	// directories, per os.Stat()) or uncompressed layer blobs which should
+	// be added to the image at commit-time.  The order of these relative
+	// to PrependedEmptyLayers and AppendedEmptyLayers, and relative to the
+	// corresponding members in the Builder object, in the committed image
+	// is not guaranteed.
+	PrependedLinkedLayers, AppendedLinkedLayers []LinkedLayer
+}
+
+// LinkedLayer combines a history entry with the location of either a directory
+// tree (if it's a directory, per os.Stat()) or an uncompressed layer blob
+// which should be added to the image at commit-time.  The BlobPath and
+// History.EmptyLayer fields should be considered mutually-exclusive.
+type LinkedLayer struct {
+	History  v1.History // history entry to add
+	BlobPath string     // corresponding uncompressed blob file (layer as a tar archive), or directory tree to archive
 }
 
 var (
@@ -348,6 +367,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 			if options.ExtraImageContent == nil {
 				options.ExtraImageContent = make(map[string]string, len(extraImageContent))
 			}
+			// merge in the scanner-generated content
 			for k, v := range extraImageContent {
 				if _, set := options.ExtraImageContent[k]; !set {
 					options.ExtraImageContent[k] = v

commit_test.go

@@ -0,0 +1,232 @@
+package buildah
+
+import (
+	"archive/tar"
+	"context"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	imageStorage "github.com/containers/image/v5/storage"
+	"github.com/containers/image/v5/types"
+	"github.com/containers/storage"
+	storageTypes "github.com/containers/storage/types"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCommitLinkedLayers(t *testing.T) {
+	ctx := context.TODO()
+	systemContext := types.SystemContext{}
+	now := time.Now()
+
+	graphDriverName := os.Getenv("STORAGE_DRIVER")
+	if graphDriverName == "" {
+		graphDriverName = "vfs"
+	}
+	store, err := storage.GetStore(storageTypes.StoreOptions{
+		RunRoot:         t.TempDir(),
+		GraphRoot:       t.TempDir(),
+		GraphDriverName: graphDriverName,
+	})
+	require.NoError(t, err, "initializing storage")
+	t.Cleanup(func() { _, err := store.Shutdown(true); assert.NoError(t, err) })
+
+	imageName := func(i int) string { return fmt.Sprintf("image%d", i) }
+	makeFile := func(base string, size int64) string {
+		t.Helper()
+		fn := filepath.Join(t.TempDir(), base)
+		f, err := os.Create(fn)
+		require.NoError(t, err)
+		defer f.Close()
+		if size == 0 {
+			size = 512
+		}
+		_, err = io.CopyN(f, rand.Reader, size)
+		require.NoErrorf(t, err, "writing payload file %d", base)
+		return f.Name()
+	}
+	makeArchive := func(base string, size int64) string {
+		t.Helper()
+		file := makeFile(base, size)
+		archiveDir := t.TempDir()
+		st, err := os.Stat(file)
+		require.NoError(t, err)
+		archiveName := filepath.Join(archiveDir, filepath.Base(file))
+		f, err := os.Create(archiveName)
+		require.NoError(t, err)
+		defer f.Close()
+		tw := tar.NewWriter(f)
+		defer tw.Close()
+		hdr, err := tar.FileInfoHeader(st, "")
+		require.NoErrorf(t, err, "building tar header for %s", file)
+		err = tw.WriteHeader(hdr)
+		require.NoErrorf(t, err, "writing tar header for %s", file)
+		f, err = os.Open(file)
+		require.NoError(t, err)
+		defer f.Close()
+		_, err = io.Copy(tw, f)
+		require.NoErrorf(t, err, "writing tar payload for %s", file)
+		return archiveName
+	}
+	layerNumber := 0
+
+	// Build a from-scratch image with one layer.
+	builderOptions := BuilderOptions{
+		FromImage: "scratch",
+		NamespaceOptions: []NamespaceOption{{
+			Name: string(rspec.NetworkNamespace),
+			Host: true,
+		}},
+	}
+	b, err := NewBuilder(ctx, store, builderOptions)
+	require.NoError(t, err, "creating builder")
+	b.SetCreatedBy(imageName(layerNumber))
+	firstFile := makeFile("file0", 0)
+	err = b.Add("/", false, AddAndCopyOptions{}, firstFile)
+	require.NoError(t, err, "adding", firstFile)
+	commitOptions := CommitOptions{}
+	ref, err := imageStorage.Transport.ParseStoreReference(store, imageName(layerNumber))
+	require.NoError(t, err, "parsing reference for to-be-committed image", imageName(layerNumber))
+	_, _, _, err = b.Commit(ctx, ref, commitOptions)
+	require.NoError(t, err, "committing", imageName(layerNumber))
+
+	// Build another image based on the first with not much in its layer.
+	builderOptions.FromImage = imageName(layerNumber)
+	layerNumber++
+	b, err = NewBuilder(ctx, store, builderOptions)
+	require.NoError(t, err, "creating builder")
+	b.SetCreatedBy(imageName(layerNumber))
+	secondFile := makeFile("file1", 0)
+	err = b.Add("/", false, AddAndCopyOptions{}, secondFile)
+	require.NoError(t, err, "adding", secondFile)
+	commitOptions = CommitOptions{}
+	ref, err = imageStorage.Transport.ParseStoreReference(store, imageName(layerNumber))
+	require.NoError(t, err, "parsing reference for to-be-committed image", imageName(layerNumber))
+	_, _, _, err = b.Commit(ctx, ref, commitOptions)
+	require.NoError(t, err, "committing", imageName(layerNumber))
+
+	// Build a third image with two layers on either side of its read-write layer.
+	builderOptions.FromImage = imageName(layerNumber)
+	layerNumber++
+	b, err = NewBuilder(ctx, store, builderOptions)
+	require.NoError(t, err, "creating builder")
+	thirdFile := makeFile("file2", 0)
+	fourthArchiveFile := makeArchive("file3", 0)
+	fifthFile := makeFile("file4", 0)
+	sixthFile := makeFile("file5", 0)
+	seventhArchiveFile := makeArchive("file6", 0)
+	eighthFile := makeFile("file7", 0)
+	ninthArchiveFile := makeArchive("file8", 0)
+	err = b.Add("/", false, AddAndCopyOptions{}, sixthFile)
+	require.NoError(t, err, "adding", sixthFile)
+	b.SetCreatedBy(imageName(layerNumber + 3))
+	b.AddPrependedLinkedLayer(nil, imageName(layerNumber), "", "", filepath.Dir(thirdFile))
+	commitOptions = CommitOptions{
+		PrependedLinkedLayers: []LinkedLayer{
+			{
+				BlobPath: fourthArchiveFile,
+				History: v1.History{
+					Created:   &now,
+					CreatedBy: imageName(layerNumber + 1),
+				},
+			},
+			{
+				BlobPath: filepath.Dir(fifthFile),
+				History: v1.History{
+					Created:   &now,
+					CreatedBy: imageName(layerNumber + 2),
+				},
+			},
+		},
+		AppendedLinkedLayers: []LinkedLayer{
+			{
+				BlobPath: seventhArchiveFile,
+				History: v1.History{
+					Created:   &now,
+					CreatedBy: imageName(layerNumber + 4),
+				},
+			},
+			{
+				BlobPath: filepath.Dir(eighthFile),
+				History: v1.History{
+					Created:   &now,
+					CreatedBy: imageName(layerNumber + 5),
+				},
+			},
+		},
+	}
+	b.AddAppendedLinkedLayer(nil, imageName(layerNumber+6), "", "", ninthArchiveFile)
+	ref, err = imageStorage.Transport.ParseStoreReference(store, imageName(layerNumber))
+	require.NoError(t, err, "parsing reference for to-be-committed image", imageName(layerNumber))
+	_, _, _, err = b.Commit(ctx, ref, commitOptions)
+	require.NoError(t, err, "committing", imageName(layerNumber))
+
+	// Build one last image based on the previous one.
+	builderOptions.FromImage = imageName(layerNumber)
+	layerNumber += 7
+	b, err = NewBuilder(ctx, store, builderOptions)
+	require.NoError(t, err, "creating builder")
+	b.SetCreatedBy(imageName(layerNumber))
+	tenthFile := makeFile("file9", 0)
+	err = b.Add("/", false, AddAndCopyOptions{}, tenthFile)
+	require.NoError(t, err, "adding", tenthFile)
+	commitOptions = CommitOptions{}
+	ref, err = imageStorage.Transport.ParseStoreReference(store, imageName(layerNumber))
+	require.NoError(t, err, "parsing reference for to-be-committed image", imageName(layerNumber))
+	_, _, _, err = b.Commit(ctx, ref, commitOptions)
+	require.NoError(t, err, "committing", imageName(layerNumber))
+
+	// Get set to examine this image.  At this point, each history entry
+	// should just have "image%d" as its CreatedBy field, and each layer
+	// should have the corresponding file (and nothing else) in it.
+	src, err := ref.NewImageSource(ctx, &systemContext)
+	require.NoError(t, err, "opening image source")
+	defer src.Close()
+	img, err := ref.NewImage(ctx, &systemContext)
+	require.NoError(t, err, "opening image")
+	defer img.Close()
+	config, err := img.OCIConfig(ctx)
+	require.NoError(t, err, "reading config in OCI format")
+	require.Len(t, config.History, 10, "history length")
+	for i := range config.History {
+		require.Equal(t, fmt.Sprintf("image%d", i), config.History[i].CreatedBy, "history createdBy is off")
+	}
+	require.Len(t, config.RootFS.DiffIDs, 10, "diffID list")
+
+	layerContents := func(archive io.ReadCloser) []string {
+		var contents []string
+		defer archive.Close()
+		tr := tar.NewReader(archive)
+		entry, err := tr.Next()
+		for entry != nil {
+			contents = append(contents, entry.Name)
+			if err != nil {
+				break
+			}
+			entry, err = tr.Next()
+		}
+		require.ErrorIs(t, err, io.EOF)
+		return contents
+	}
+	infos, err := img.LayerInfosForCopy(ctx)
+	require.NoError(t, err, "getting layer infos")
+	require.Len(t, infos, 10)
+	for i, blobInfo := range infos {
+		func() {
+			t.Helper()
+			rc, _, err := src.GetBlob(ctx, blobInfo, nil)
+			require.NoError(t, err, "getting blob", i)
+			defer rc.Close()
+			contents := layerContents(rc)
+			require.Len(t, contents, 1)
+			require.Equal(t, fmt.Sprintf("file%d", i), contents[0])
+		}()
+	}
+}

config.go

@@ -753,3 +753,62 @@ func (b *Builder) AddAppendedEmptyLayer(created *time.Time, createdBy, author, c
 func (b *Builder) ClearAppendedEmptyLayers() {
 	b.AppendedEmptyLayers = nil
 }
+
+// AddPrependedLinkedLayer adds an item to the history that we'll create when
+// committing the image, optionally with a layer, after any history we inherit
+// from the base image, but before the history item that we'll use to describe
+// the new layer that we're adding.
+// The blobPath can be either the location of an uncompressed archive, or a
+// directory whose contents will be archived to use as a layer blob.  Leaving
+// blobPath empty is functionally similar to calling AddPrependedEmptyLayer().
+func (b *Builder) AddPrependedLinkedLayer(created *time.Time, createdBy, author, comment, blobPath string) {
+	if created != nil {
+		copiedTimestamp := *created
+		created = &copiedTimestamp
+	}
+	b.PrependedLinkedLayers = append(b.PrependedLinkedLayers, LinkedLayer{
+		BlobPath: blobPath,
+		History: ociv1.History{
+			Created:    created,
+			CreatedBy:  createdBy,
+			Author:     author,
+			Comment:    comment,
+			EmptyLayer: blobPath == "",
+		},
+	})
+}
+
+// ClearPrependedLinkedLayers clears the list of history entries that we'll add
+// the committed image before the layer that we're adding (if we're adding it).
+func (b *Builder) ClearPrependedLinkedLayers() {
+	b.PrependedLinkedLayers = nil
+}
+
+// AddAppendedLinkedLayer adds an item to the history that we'll create when
+// committing the image, optionally with a layer, after the history item that
+// we'll use to describe the new layer that we're adding.
+// The blobPath can be either the location of an uncompressed archive, or a
+// directory whose contents will be archived to use as a layer blob.  Leaving
+// blobPath empty is functionally similar to calling AddAppendedEmptyLayer().
+func (b *Builder) AddAppendedLinkedLayer(created *time.Time, createdBy, author, comment, blobPath string) {
+	if created != nil {
+		copiedTimestamp := *created
+		created = &copiedTimestamp
+	}
+	b.AppendedLinkedLayers = append(b.AppendedLinkedLayers, LinkedLayer{
+		BlobPath: blobPath,
+		History: ociv1.History{
+			Created:    created,
+			CreatedBy:  createdBy,
+			Author:     author,
+			Comment:    comment,
+			EmptyLayer: blobPath == "",
+		},
+	})
+}
+
+// ClearAppendedLinkedLayers clears the list of linked layers that we'll add to
+// the committed image after the layer that we're adding (if we're adding it).
+func (b *Builder) ClearAppendedLinkedLayers() {
+	b.AppendedLinkedLayers = nil
+}