diff --git a/go.mod b/go.mod index 44486a9..aaa9248 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ toolchain go1.23.0 require ( github.com/BurntSushi/toml v1.4.0 - github.com/KimMachineGun/automemlimit v0.6.1 github.com/MakeNowJust/heredoc v1.0.0 github.com/Masterminds/sprig/v3 v3.3.0 github.com/Shopify/ejson v1.5.2 @@ -35,13 +34,9 @@ require ( github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cilium/ebpf v0.9.1 // indirect github.com/cloudfoundry-community/vaultkv v0.7.0 // indirect - github.com/containerd/cgroups/v3 v3.0.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/docker/go-units v0.4.0 // indirect github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -52,7 +47,6 @@ require ( github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect - github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect @@ -84,15 +78,12 @@ require ( github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/gomega v1.33.1 // indirect - github.com/opencontainers/runtime-spec v1.0.2 // indirect - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect - github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect diff --git a/go.sum b/go.sum index ab75b6d..adb4f0e 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/KimMachineGun/automemlimit v0.6.1 h1:ILa9j1onAAMadBsyyUJv5cack8Y1WT26yLj/V+ulKp8= -github.com/KimMachineGun/automemlimit v0.6.1/go.mod h1:T7xYht7B8r6AG/AqFcUdc7fzd2bIdBKmepfP2S1svPY= github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -67,18 +65,13 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4= -github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudfoundry-community/vaultkv v0.7.0 h1:VFq0TQxGIxuJuqXKDlY73XneOQyKKTEBA8EwqKI3OOU= github.com/cloudfoundry-community/vaultkv v0.7.0/go.mod h1:D17jAL9n2GS66nbapOU7vRkGQ2D5zhsnyhCuspfNDlg= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/containerd/cgroups/v3 v3.0.1 h1:4hfGvu8rfGIwVIDd+nLzn/B9ZXx4BcCjzt5ToenJRaE= -github.com/containerd/cgroups/v3 v3.0.1/go.mod h1:/vtwk1VXrtoa5AaZLkypuOJgA/6DyPMZHJPGQNtlHnw= github.com/coreos/go-oidc/v3 v3.5.0/go.mod h1:ecXRtV4romGPeO6ieExAsUK9cb/3fp9hXNz1tlv8PIM= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -87,8 +80,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad h1:Qk76DOWdOp+GlyDKBAG3Klr9cn7N+LcYc82AZ2S7+cA= github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad/go.mod h1:mPKfmRa823oBIgl2r20LeMSpTAteW5j7FLkc0vjmzyQ= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= @@ -135,7 +126,6 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8Wd github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -309,10 +299,6 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= -github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= -github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= @@ -341,8 +327,6 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smarty/assertions v1.16.0 h1:EvHNkdRA4QHMrn75NZSoUQ/mAUXAYWfatfB01yTCzfY= github.com/smarty/assertions v1.16.0/go.mod h1:duaaFdCS0K9dnoM50iyek/eYINOZ64gbh1Xlf6LG7AI= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= @@ -533,7 +517,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/subst/build.go b/pkg/subst/build.go index af6b7b8..0d2eb3a 100644 --- a/pkg/subst/build.go +++ b/pkg/subst/build.go @@ -3,7 +3,6 @@ package subst import ( "context" "fmt" - "sync" decrypt "github.com/bedag/subst/internal/decryptors" ejson "github.com/bedag/subst/internal/decryptors/ejson" @@ -89,57 +88,44 @@ func (b *Build) Build() (err error) { // Run Build log.Debug().Msg("substitute manifests") - var wg sync.WaitGroup - manifestsMutex := sync.Mutex{} for _, manifest := range b.Substitutions.Resources.Resources() { - wg.Add(1) - go func() { - defer wg.Done() + var c map[interface{}]interface{} - var c map[interface{}]interface{} - - mBytes, _ := manifest.MarshalJSON() - // should not check every file if its encrypted - // already decrypted in substiqutions.go? - for _, d := range decryptors { - isEncrypted, err := d.IsEncrypted(mBytes) - if err != nil { - log.Error().Msgf("Error checking encryption for %s: %s", mBytes, err) - continue - } - if isEncrypted { - dm, err := d.Decrypt(mBytes) - if err != nil { - log.Error().Msgf("failed to decrypt %s: %s", mBytes, err) - return - } - c = utils.ToInterface(dm) - break - } + mBytes, _ := manifest.MarshalJSON() + for _, d := range decryptors { + isEncrypted, err := d.IsEncrypted(mBytes) + if err != nil { + log.Error().Msgf("Error checking encryption for %s: %s", mBytes, err) + return err } - - if c == nil { - m, _ := manifest.AsYAML() - - c, err = utils.ParseYAML(m) + if isEncrypted { + dm, err := d.Decrypt(mBytes) if err != nil { - log.Error().Msgf("UnmarshalJSON: %s", err) - return + log.Error().Msgf("failed to decrypt %s: %s", mBytes, err) + return err } + c = utils.ToInterface(dm) + break } + } - f, err := b.Substitutions.Eval(c, nil, false) + if c == nil { + m, _ := manifest.AsYAML() + + c, err = utils.ParseYAML(m) if err != nil { - log.Error().Msgf("spruce evaluation failed %s/%s: %s", manifest.GetNamespace(), manifest.GetName(), err) - return + log.Error().Msgf("UnmarshalJSON: %s", err) + return err } - manifestsMutex.Lock() - b.Manifests = append(b.Manifests, f) - manifestsMutex.Unlock() - }() - } + } - wg.Wait() + f, err := b.Substitutions.Eval(c, nil, false) + if err != nil { + log.Error().Msgf("spruce evaluation failed %s/%s: %s", manifest.GetNamespace(), manifest.GetName(), err) + return err + } + b.Manifests = append(b.Manifests, f) + } return nil } diff --git a/subst/cmd/root.go b/subst/cmd/root.go index 3abf577..4be4dad 100644 --- a/subst/cmd/root.go +++ b/subst/cmd/root.go @@ -2,16 +2,14 @@ package cmd import ( "fmt" - "log" - "log/slog" "os" "path/filepath" "runtime/pprof" "strconv" "sync" - "github.com/KimMachineGun/automemlimit/memlimit" "github.com/rs/zerolog" + "github.com/rs/zerolog/log" flag "github.com/spf13/pflag" "github.com/MakeNowJust/heredoc" @@ -23,7 +21,6 @@ import ( var ( cfgFile string v string - m float64 p int cpuProfile bool memProfile bool @@ -46,9 +43,6 @@ func NewRootCmd() *cobra.Command { if err := setUpLogs(v); err != nil { return err } - if err := setUpMemLimitRatio(m); err != nil { - return err - } if err := setUpMaxProcs(p); err != nil { return err } @@ -59,8 +53,6 @@ func NewRootCmd() *cobra.Command { //Default value is the warn level cmd.PersistentFlags().StringVarP(&v, "verbosity", "v", zerolog.WarnLevel.String(), "Log level (debug, info, warn, error, fatal, panic") - //Default value is 0.1 (10%) - cmd.PersistentFlags().Float64VarP(&m, "memlimitratio", "m", 0.1, "Overwrite GOMEMLIMIT which the command can allocate (default: 0.1 which means 10%)") //Default value is inferred from cgroups or system cmd.PersistentFlags().IntVarP(&p, "maxprocs", "p", 0, "Overwrite GOMAXPROCS for the command to use (default: 0 which means respect cgroup or system)") @@ -101,30 +93,12 @@ func setUpLogs(level string) error { return nil } -// setUpMemLimitRatio set the memlimit ratio -func setUpMemLimitRatio(ratio float64) error { - _, err := memlimit.SetGoMemLimitWithOpts( - memlimit.WithRatio(ratio), - memlimit.WithProvider( - memlimit.ApplyFallback( - memlimit.FromCgroup, - memlimit.FromSystem, - ), - ), - memlimit.WithLogger(slog.Default()), - ) - if err != nil { - return err - } - return nil -} - // setUpMaxProcs set the max procs func setUpMaxProcs(procs int) error { if procs > 0 { os.Setenv("GOMAXPROCS", strconv.Itoa(procs)) } - _, err := maxprocs.Set(maxprocs.Logger(log.Printf)) + _, err := maxprocs.Set(maxprocs.Logger(log.Debug().Msgf)) if err != nil { return err } diff --git a/subst/cmd/root_test.go b/subst/cmd/root_test.go index bead637..7ead875 100644 --- a/subst/cmd/root_test.go +++ b/subst/cmd/root_test.go @@ -13,7 +13,7 @@ func BenchmarkExecute(b *testing.B) { "/Users/adrian/git/inventory/clusters/k8s-bedag-root-dev", }) for i := 0; i < b.N; i++ { - debug.SetGCPercent(800) + debug.SetGCPercent(100) err := cmd.Execute() if err != nil { b.Errorf("Error: %v", err)