From b755b72dbb6e3adb3f0818ca4c5b6cd678639d66 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 4 Oct 2024 16:12:17 +0200 Subject: [PATCH] feat(directives): allow ignore rules for `copy` (#2649) Signed-off-by: Hidde Beydals (cherry picked from commit 694504dbc431233f8cfb1477ec18affa212123c6) --- go.mod | 5 + go.sum | 14 +- internal/directives/file_copier.go | 55 +++++ internal/directives/file_copier_test.go | 217 ++++++++++++++++++ ...directive-config.json => copy-config.json} | 5 + internal/directives/zz_config_types.go | 3 + 6 files changed, 297 insertions(+), 2 deletions(-) rename internal/directives/schemas/{copy-directive-config.json => copy-config.json} (69%) diff --git a/go.mod b/go.mod index 6812b3357..596c13d45 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.0 github.com/fatih/structtag v1.2.0 github.com/fluxcd/pkg/kustomize v1.13.0 + github.com/go-git/go-git/v5 v5.12.0 github.com/gogo/protobuf v1.3.2 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/go-containerregistry v0.20.2 @@ -88,6 +89,8 @@ require ( github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fatih/color v1.16.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/uuid v4.0.0+incompatible // indirect @@ -100,6 +103,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.5.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -121,6 +125,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.28.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect k8s.io/component-base v0.31.1 // indirect k8s.io/kubectl v0.31.0 // indirect ) diff --git a/go.sum b/go.sum index e2e61f347..dbc0e304d 100644 --- a/go.sum +++ b/go.sum @@ -185,6 +185,12 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= github.com/go-jose/go-jose/v4 v4.0.2 h1:R3l3kkBds16bO7ZFAEEcofK0MkrAJt3jlJznWZG0nvk= @@ -313,6 +319,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jferrl/go-githubauth v1.1.1 h1:HfF3eeWFL+9jV9KHAatBaEnFGm9R2LkTqo5Z2GcDk20= github.com/jferrl/go-githubauth v1.1.1/go.mod h1:FC1jqgik3xdaZDg8CUmGbvDwfP/egXkrq6Ygl9pSz/Y= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -465,8 +473,8 @@ github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlX github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -695,6 +703,8 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/internal/directives/file_copier.go b/internal/directives/file_copier.go index cba02c1fa..ecbd088ad 100644 --- a/internal/directives/file_copier.go +++ b/internal/directives/file_copier.go @@ -1,14 +1,17 @@ package directives import ( + "bufio" "context" "errors" "fmt" "io/fs" + "os" "path/filepath" "strings" securejoin "github.com/cyphar/filepath-securejoin" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" "github.com/otiai10/copy" "github.com/xeipuuv/gojsonschema" @@ -80,12 +83,22 @@ func (f *fileCopier) runPromotionStep( fmt.Errorf("could not secure join outPath %q: %w", cfg.OutPath, err) } + // Load the ignore rules. + matcher, err := f.loadIgnoreRules(inPath, cfg.Ignore) + if err != nil { + return PromotionStepResult{Status: kargoapi.PromotionPhaseErrored}, + fmt.Errorf("failed to load ignore rules: %w", err) + } + // Perform the copy operation. opts := copy.Options{ OnSymlink: func(src string) copy.SymlinkAction { logging.LoggerFromContext(ctx).Trace("ignoring symlink", "src", src) return copy.Skip }, + Skip: func(f os.FileInfo, src, _ string) (bool, error) { + return matcher.Match(strings.Split(src, string(filepath.Separator)), f.IsDir()), nil + }, OnError: func(_, _ string, err error) error { return sanitizePathError(err, stepCtx.WorkDir) }, @@ -97,6 +110,48 @@ func (f *fileCopier) runPromotionStep( return PromotionStepResult{Status: kargoapi.PromotionPhaseSucceeded}, nil } +// loadIgnoreRules loads the ignore rules from the given string. The rules are +// separated by newlines, and comments are allowed with the '#' character. +// It returns a gitignore.Matcher that can be used to match paths against the +// rules. +func (f *fileCopier) loadIgnoreRules(inPath, rules string) (gitignore.Matcher, error) { + // Determine the domain for the ignore rules. For directories, the domain is + // the directory itself. For files, the domain is the parent directory. + fi, err := os.Lstat(inPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + // Let the error slide if the path does not exist, to allow + // the copy operation to fail later. This provides a more + // predictable user experience. + return gitignore.NewMatcher(nil), nil + } + return nil, fmt.Errorf("failed to determine domain: %w", err) + } + var domain []string + switch { + case fi.IsDir(): + domain = strings.Split(inPath, string(filepath.Separator)) + default: + domain = strings.Split(filepath.Dir(inPath), string(filepath.Separator)) + } + + // Default patterns to ignore the .git directory. + ps := []gitignore.Pattern{ + gitignore.ParsePattern(".git", domain), + } + + // Parse additional user-provided rules. + scanner := bufio.NewScanner(strings.NewReader(rules)) + for scanner.Scan() { + s := scanner.Text() + if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 { + ps = append(ps, gitignore.ParsePattern(s, domain)) + } + } + + return gitignore.NewMatcher(ps), nil +} + // sanitizePathError sanitizes the path in a path error to be relative to the // work directory. If the path cannot be made relative, the filename is used // instead. diff --git a/internal/directives/file_copier_test.go b/internal/directives/file_copier_test.go index ed975f7be..e4dcf3e25 100644 --- a/internal/directives/file_copier_test.go +++ b/internal/directives/file_copier_test.go @@ -5,8 +5,10 @@ import ( "errors" "os" "path/filepath" + "strings" "testing" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -121,6 +123,117 @@ func Test_fileCopier_runPromotionStep(t *testing.T) { assert.True(t, os.IsNotExist(err)) }, }, + { + name: "default ignore rule ignores .git directory", + setupFiles: func(t *testing.T) string { + tmpDir := t.TempDir() + + inDir := filepath.Join(tmpDir, "input") + require.NoError(t, os.Mkdir(inDir, 0o755)) + + ignoreFilePath := filepath.Join(inDir, ".git", "file.txt") + require.NoError(t, os.MkdirAll(filepath.Dir(ignoreFilePath), 0o755)) + require.NoError(t, os.WriteFile(ignoreFilePath, []byte("ignored content"), 0o600)) + + filePath := filepath.Join(inDir, "input.txt") + require.NoError(t, os.WriteFile(filePath, []byte("test content"), 0o600)) + + return tmpDir + }, + cfg: CopyConfig{ + InPath: "input/", + OutPath: "output/", + }, + assertions: func(t *testing.T, workDir string, result PromotionStepResult, err error) { + assert.NoError(t, err) + require.Equal(t, PromotionStepResult{Status: kargoapi.PromotionPhaseSucceeded}, result) + + outDir := filepath.Join(workDir, "output") + + outPath := filepath.Join(outDir, "input.txt") + b, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Equal(t, "test content", string(b)) + + ignoreFilePath := filepath.Join(outDir, ".git", "file.txt") + require.NoDirExists(t, filepath.Dir(ignoreFilePath)) + require.NoFileExists(t, ignoreFilePath) + }, + }, + { + name: "default ignore rule ignores .git file", + setupFiles: func(t *testing.T) string { + tmpDir := t.TempDir() + + inDir := filepath.Join(tmpDir, "input") + require.NoError(t, os.Mkdir(inDir, 0o755)) + + ignoreFilePath := filepath.Join(inDir, ".git") + require.NoError(t, os.WriteFile(ignoreFilePath, []byte("ignored content"), 0o600)) + + filePath := filepath.Join(inDir, "input.txt") + require.NoError(t, os.WriteFile(filePath, []byte("test content"), 0o600)) + + return tmpDir + }, + cfg: CopyConfig{ + InPath: "input/", + OutPath: "output/", + }, + assertions: func(t *testing.T, workDir string, result PromotionStepResult, err error) { + assert.NoError(t, err) + require.Equal(t, PromotionStepResult{Status: kargoapi.PromotionPhaseSucceeded}, result) + + outDir := filepath.Join(workDir, "output") + + outPath := filepath.Join(outDir, "input.txt") + b, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Equal(t, "test content", string(b)) + + ignoreFilePath := filepath.Join(outDir, ".git") + require.NoFileExists(t, ignoreFilePath) + }, + }, + { + name: "overwrite default ignore rules", + setupFiles: func(t *testing.T) string { + tmpDir := t.TempDir() + + inDir := filepath.Join(tmpDir, "input") + require.NoError(t, os.Mkdir(inDir, 0o755)) + + ignoreFilePath := filepath.Join(inDir, ".git", "file.txt") + require.NoError(t, os.MkdirAll(filepath.Dir(ignoreFilePath), 0o755)) + require.NoError(t, os.WriteFile(ignoreFilePath, []byte("included content"), 0o600)) + + filePath := filepath.Join(inDir, "input.txt") + require.NoError(t, os.WriteFile(filePath, []byte("test content"), 0o600)) + + return tmpDir + }, + cfg: CopyConfig{ + InPath: "input/", + OutPath: "output/", + Ignore: "!.git/", + }, + assertions: func(t *testing.T, workDir string, result PromotionStepResult, err error) { + assert.NoError(t, err) + require.Equal(t, PromotionStepResult{Status: kargoapi.PromotionPhaseSucceeded}, result) + + outDir := filepath.Join(workDir, "output") + + outPath := filepath.Join(outDir, "input.txt") + b, err := os.ReadFile(outPath) + require.NoError(t, err) + assert.Equal(t, "test content", string(b)) + + ignoreFilePath := filepath.Join(outDir, ".git", "file.txt") + b, err = os.ReadFile(ignoreFilePath) + require.NoError(t, err) + assert.Equal(t, "included content", string(b)) + }, + }, { name: "fails with invalid input path", setupFiles: func(t *testing.T) string { @@ -151,6 +264,110 @@ func Test_fileCopier_runPromotionStep(t *testing.T) { } } +func Test_fileCopier_loadIgnoreRules(t *testing.T) { + tests := []struct { + name string + inPath string + rules string + setup func(*testing.T) string + assertions func(*testing.T, string, gitignore.Matcher, error) + }{ + { + name: "directory path", + inPath: "testdir", + rules: `*.txt +# comment +*.go`, + setup: func(t *testing.T) string { + return t.TempDir() + }, + assertions: func(t *testing.T, inPath string, matcher gitignore.Matcher, err error) { + assert.NoError(t, err) + assert.NotNil(t, matcher) + + basePath := strings.Split(inPath, string(filepath.Separator)) + + // Provided rules + assert.True(t, matcher.Match(append(basePath, "file.txt"), false)) + assert.True(t, matcher.Match(append(basePath, "file.go"), false)) + assert.False(t, matcher.Match(append(basePath, "file.log"), false)) + + // Default rules + assert.True(t, matcher.Match(append(basePath, ".git"), true)) + assert.True(t, matcher.Match(append(basePath, ".git", "file.log"), true)) + }, + }, + { + name: "file path", + rules: "*.log", + setup: func(t *testing.T) string { + dir := t.TempDir() + inPath := filepath.Join(dir, "testfile.txt") + assert.NoError(t, os.WriteFile(inPath, []byte("test"), 0o600)) + return inPath + }, + assertions: func(t *testing.T, inPath string, matcher gitignore.Matcher, err error) { + assert.NoError(t, err) + assert.NotNil(t, matcher) + + basePath := strings.Split(filepath.Dir(inPath), string(filepath.Separator)) + assert.True(t, matcher.Match(append(basePath, "something.log"), false)) + assert.False(t, matcher.Match(append(basePath, "testfile.txt"), false)) + assert.True(t, matcher.Match(append(basePath, ".git", "file"), false)) + }, + }, + { + name: "non-existent path", + inPath: "nonexistent", + rules: "*.tmp", + setup: func(*testing.T) string { + return "does-not-exist" + }, + assertions: func(t *testing.T, _ string, matcher gitignore.Matcher, err error) { + assert.NoError(t, err) + assert.NotNil(t, matcher) + }, + }, + { + name: "empty rules", + inPath: "testdir", + rules: "", + setup: func(t *testing.T) string { + return t.TempDir() + }, + assertions: func(t *testing.T, inPath string, matcher gitignore.Matcher, err error) { + assert.NoError(t, err) + assert.NotNil(t, matcher) + + basePath := strings.Split(inPath, string(filepath.Separator)) + assert.True(t, matcher.Match(append(basePath, ".git", "file"), false)) + assert.False(t, matcher.Match(append(basePath, "file.txt"), false)) + }, + }, + { + name: "invalid path", + setup: func(*testing.T) string { + return string([]byte{0}) + }, + rules: "*.tmp", + assertions: func(t *testing.T, _ string, matcher gitignore.Matcher, err error) { + assert.ErrorContains(t, err, "failed to determine domain") + assert.Nil(t, matcher) + }, + }, + } + + for _, tt := range tests { + f := &fileCopier{} + + t.Run(tt.name, func(t *testing.T) { + inPath := tt.setup(t) + matcher, err := f.loadIgnoreRules(inPath, tt.rules) + tt.assertions(t, inPath, matcher, err) + }) + } +} + func Test_sanitizePathError(t *testing.T) { tests := []struct { name string diff --git a/internal/directives/schemas/copy-directive-config.json b/internal/directives/schemas/copy-config.json similarity index 69% rename from internal/directives/schemas/copy-directive-config.json rename to internal/directives/schemas/copy-config.json index ce7a6f591..9d31408b0 100644 --- a/internal/directives/schemas/copy-directive-config.json +++ b/internal/directives/schemas/copy-config.json @@ -14,6 +14,11 @@ "type": "string", "description": "OutPath is the path to the destination file or directory.", "minLength": 1 + }, + "ignore": { + "type": "string", + "description": "Ignore is a (multiline) string of glob patterns to ignore when copying files. It accepts the same syntax as .gitignore files.", + "minLength": 1 } } } diff --git a/internal/directives/zz_config_types.go b/internal/directives/zz_config_types.go index 0ede47bd3..e8b1f4caa 100644 --- a/internal/directives/zz_config_types.go +++ b/internal/directives/zz_config_types.go @@ -97,6 +97,9 @@ type ArgoCDKustomizeImageUpdate struct { } type CopyConfig struct { + // Ignore is a (multiline) string of glob patterns to ignore when copying files. It accepts + // the same syntax as .gitignore files. + Ignore string `json:"ignore,omitempty"` // InPath is the path to the file or directory to copy. InPath string `json:"inPath"` // OutPath is the path to the destination file or directory.