diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c57accd9a..b3ccd5c05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,6 +18,11 @@ jobs: - run: make build env: GOPATH: ${{ runner.workspace }} + - name: Save binary + uses: actions/upload-artifact@v1 + with: + name: gomplate + path: bin/gomplate - run: make test env: GOPATH: ${{ runner.workspace }} @@ -41,6 +46,11 @@ jobs: - run: make build env: GOPATH: ${{ runner.workspace }} + - name: Save binary + uses: actions/upload-artifact@v1 + with: + name: gomplate.exe + path: bin/gomplate.exe - run: make test env: GOPATH: ${{ runner.workspace }} diff --git a/cmd/gomplate/config.go b/cmd/gomplate/config.go new file mode 100644 index 000000000..022786181 --- /dev/null +++ b/cmd/gomplate/config.go @@ -0,0 +1,253 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/hairyhenderson/gomplate/v3/conv" + "github.com/hairyhenderson/gomplate/v3/env" + "github.com/hairyhenderson/gomplate/v3/internal/config" + + "github.com/rs/zerolog" + + "github.com/spf13/afero" + "github.com/spf13/cobra" +) + +const ( + defaultConfigFile = ".gomplate.yaml" +) + +var fs = afero.NewOsFs() + +// loadConfig is intended to be called before command execution. It: +// - creates a config.Config from the cobra flags +// - creates a config.Config from the config file (if present) +// - merges the two (flags take precedence) +// - validates the final config +// - converts the config to a *gomplate.Config for further use (TODO: eliminate this part) +func loadConfig(cmd *cobra.Command, args []string) (*config.Config, error) { + ctx := cmd.Context() + flagConfig, err := cobraConfig(cmd, args) + if err != nil { + return nil, err + } + + cfg, err := readConfigFile(cmd) + if err != nil { + return nil, err + } + if cfg == nil { + cfg = flagConfig + } else { + cfg = cfg.MergeFrom(flagConfig) + } + + cfg, err = applyEnvVars(ctx, cfg) + if err != nil { + return nil, err + } + + // reset defaults before validation + cfg.ApplyDefaults() + + err = cfg.Validate() + if err != nil { + return nil, fmt.Errorf("failed to validate merged config: %w\n%+v", err, cfg) + } + return cfg, nil +} + +func pickConfigFile(cmd *cobra.Command) (cfgFile string, required bool) { + cfgFile = defaultConfigFile + if c := env.Getenv("GOMPLATE_CONFIG"); c != "" { + cfgFile = c + required = true + } + if cmd.Flags().Changed("config") && cmd.Flag("config").Value.String() != "" { + // Use config file from the flag if specified + cfgFile = cmd.Flag("config").Value.String() + required = true + } + return cfgFile, required +} + +func readConfigFile(cmd *cobra.Command) (cfg *config.Config, err error) { + ctx := cmd.Context() + if ctx == nil { + ctx = context.Background() + } + log := zerolog.Ctx(ctx) + + cfgFile, configRequired := pickConfigFile(cmd) + + f, err := fs.Open(cfgFile) + if err != nil { + if configRequired { + return cfg, fmt.Errorf("config file requested, but couldn't be opened: %w", err) + } + return nil, nil + } + + cfg, err = config.Parse(f) + if err != nil && configRequired { + return cfg, fmt.Errorf("config file requested, but couldn't be parsed: %w", err) + } + + log.Debug().Str("cfgFile", cfgFile).Msg("using config file") + + return cfg, err +} + +// cobraConfig - initialize a config from the commandline options +func cobraConfig(cmd *cobra.Command, args []string) (cfg *config.Config, err error) { + cfg = &config.Config{} + cfg.InputFiles, err = getStringSlice(cmd, "file") + if err != nil { + return nil, err + } + cfg.Input, err = getString(cmd, "in") + if err != nil { + return nil, err + } + cfg.InputDir, err = getString(cmd, "input-dir") + if err != nil { + return nil, err + } + + cfg.ExcludeGlob, err = getStringSlice(cmd, "exclude") + if err != nil { + return nil, err + } + includesFlag, err := getStringSlice(cmd, "include") + if err != nil { + return nil, err + } + // support --include + cfg.ExcludeGlob = processIncludes(includesFlag, cfg.ExcludeGlob) + + cfg.OutputFiles, err = getStringSlice(cmd, "out") + if err != nil { + return nil, err + } + cfg.Templates, err = getStringSlice(cmd, "template") + if err != nil { + return nil, err + } + cfg.OutputDir, err = getString(cmd, "output-dir") + if err != nil { + return nil, err + } + cfg.OutputMap, err = getString(cmd, "output-map") + if err != nil { + return nil, err + } + cfg.OutMode, err = getString(cmd, "chmod") + if err != nil { + return nil, err + } + + if len(args) > 0 { + cfg.PostExec = args + } + + cfg.ExecPipe, err = getBool(cmd, "exec-pipe") + if err != nil { + return nil, err + } + + cfg.LDelim, err = getString(cmd, "left-delim") + if err != nil { + return nil, err + } + cfg.RDelim, err = getString(cmd, "right-delim") + if err != nil { + return nil, err + } + + ds, err := getStringSlice(cmd, "datasource") + if err != nil { + return nil, err + } + cx, err := getStringSlice(cmd, "context") + if err != nil { + return nil, err + } + hdr, err := getStringSlice(cmd, "datasource-header") + if err != nil { + return nil, err + } + err = cfg.ParseDataSourceFlags(ds, cx, hdr) + if err != nil { + return nil, err + } + + pl, err := getStringSlice(cmd, "plugin") + if err != nil { + return nil, err + } + err = cfg.ParsePluginFlags(pl) + if err != nil { + return nil, err + } + return cfg, nil +} + +func getStringSlice(cmd *cobra.Command, flag string) (s []string, err error) { + if cmd.Flag(flag) != nil && cmd.Flag(flag).Changed { + s, err = cmd.Flags().GetStringSlice(flag) + } + return s, err +} + +func getString(cmd *cobra.Command, flag string) (s string, err error) { + if cmd.Flag(flag) != nil && cmd.Flag(flag).Changed { + s, err = cmd.Flags().GetString(flag) + } + return s, err +} + +func getBool(cmd *cobra.Command, flag string) (b bool, err error) { + if cmd.Flag(flag) != nil && cmd.Flag(flag).Changed { + b, err = cmd.Flags().GetBool(flag) + } + return b, err +} + +// process --include flags - these are analogous to specifying --exclude '*', +// then the inverse of the --include options. +func processIncludes(includes, excludes []string) []string { + if len(includes) == 0 && len(excludes) == 0 { + return nil + } + + out := []string{} + // if any --includes are set, we start by excluding everything + if len(includes) > 0 { + out = make([]string, 1+len(includes)) + out[0] = "*" + } + for i, include := range includes { + // includes are just the opposite of an exclude + out[i+1] = "!" + include + } + out = append(out, excludes...) + return out +} + +func applyEnvVars(ctx context.Context, cfg *config.Config) (*config.Config, error) { + if to := env.Getenv("GOMPLATE_PLUGIN_TIMEOUT"); cfg.PluginTimeout == 0 && to != "" { + t, err := time.ParseDuration(to) + if err != nil { + return nil, fmt.Errorf("GOMPLATE_PLUGIN_TIMEOUT set to invalid value %q: %w", to, err) + } + cfg.PluginTimeout = t + } + + if !cfg.SuppressEmpty && conv.ToBool(env.Getenv("GOMPLATE_SUPPRESS_EMPTY", "false")) { + cfg.SuppressEmpty = true + } + + return cfg, nil +} diff --git a/cmd/gomplate/config_test.go b/cmd/gomplate/config_test.go new file mode 100644 index 000000000..dc05dfbbb --- /dev/null +++ b/cmd/gomplate/config_test.go @@ -0,0 +1,258 @@ +package main + +import ( + "context" + "os" + "testing" + "time" + + "github.com/hairyhenderson/gomplate/v3/internal/config" + + "github.com/spf13/afero" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" +) + +func TestReadConfigFile(t *testing.T) { + fs = afero.NewMemMapFs() + defer func() { fs = afero.NewOsFs() }() + cmd := &cobra.Command{} + + _, err := readConfigFile(cmd) + assert.NoError(t, err) + + cmd.Flags().String("config", defaultConfigFile, "foo") + + _, err = readConfigFile(cmd) + assert.NoError(t, err) + + cmd.ParseFlags([]string{"--config", "config.file"}) + + _, err = readConfigFile(cmd) + assert.Error(t, err) + + cmd = &cobra.Command{} + cmd.Flags().String("config", defaultConfigFile, "foo") + + f, err := fs.Create(defaultConfigFile) + assert.NoError(t, err) + f.WriteString("") + + cfg, err := readConfigFile(cmd) + assert.NoError(t, err) + assert.EqualValues(t, &config.Config{}, cfg) + + cmd.ParseFlags([]string{"--config", "config.yaml"}) + + f, err = fs.Create("config.yaml") + assert.NoError(t, err) + f.WriteString("in: hello world\n") + + cfg, err = readConfigFile(cmd) + assert.NoError(t, err) + assert.EqualValues(t, &config.Config{Input: "hello world"}, cfg) + + f.WriteString("in: ") + + _, err = readConfigFile(cmd) + assert.Error(t, err) +} + +func TestLoadConfig(t *testing.T) { + fs = afero.NewMemMapFs() + defer func() { fs = afero.NewOsFs() }() + + cmd := &cobra.Command{} + cmd.Args = optionalExecArgs + cmd.Flags().StringSlice("file", []string{"-"}, "...") + cmd.Flags().StringSlice("out", []string{"-"}, "...") + cmd.Flags().String("in", ".", "...") + cmd.Flags().String("output-dir", ".", "...") + cmd.Flags().String("left-delim", "{{", "...") + cmd.Flags().String("right-delim", "}}", "...") + cmd.Flags().Bool("exec-pipe", false, "...") + cmd.ParseFlags(nil) + + out, err := loadConfig(cmd, cmd.Flags().Args()) + expected := &config.Config{ + InputFiles: []string{"-"}, + OutputFiles: []string{"-"}, + LDelim: "{{", + RDelim: "}}", + PostExecInput: os.Stdin, + OutWriter: os.Stdout, + PluginTimeout: 5 * time.Second, + } + assert.NoError(t, err) + assert.EqualValues(t, expected, out) + + cmd.ParseFlags([]string{"--in", "foo"}) + out, err = loadConfig(cmd, cmd.Flags().Args()) + expected = &config.Config{ + Input: "foo", + OutputFiles: []string{"-"}, + LDelim: "{{", + RDelim: "}}", + PostExecInput: os.Stdin, + OutWriter: os.Stdout, + PluginTimeout: 5 * time.Second, + } + assert.NoError(t, err) + assert.EqualValues(t, expected, out) + + cmd.ParseFlags([]string{"--in", "foo", "--exec-pipe", "--", "tr", "[a-z]", "[A-Z]"}) + out, err = loadConfig(cmd, cmd.Flags().Args()) + expected = &config.Config{ + Input: "foo", + LDelim: "{{", + RDelim: "}}", + ExecPipe: true, + PostExec: []string{"tr", "[a-z]", "[A-Z]"}, + PostExecInput: out.PostExecInput, + OutWriter: out.PostExecInput, + OutputFiles: []string{"-"}, + PluginTimeout: 5 * time.Second, + } + assert.NoError(t, err) + assert.EqualValues(t, expected, out) +} + +func TestCobraConfig(t *testing.T) { + t.Parallel() + cmd := &cobra.Command{} + cmd.Flags().StringSlice("file", []string{"-"}, "...") + cmd.Flags().StringSlice("out", []string{"-"}, "...") + cmd.Flags().String("output-dir", ".", "...") + cmd.Flags().String("left-delim", "{{", "...") + cmd.Flags().String("right-delim", "}}", "...") + cmd.ParseFlags(nil) + + cfg, err := cobraConfig(cmd, cmd.Flags().Args()) + assert.NoError(t, err) + assert.EqualValues(t, &config.Config{}, cfg) + + cmd.ParseFlags([]string{"--file", "in", "--", "echo", "foo"}) + + cfg, err = cobraConfig(cmd, cmd.Flags().Args()) + assert.NoError(t, err) + assert.EqualValues(t, &config.Config{ + InputFiles: []string{"in"}, + PostExec: []string{"echo", "foo"}, + }, cfg) +} + +func TestProcessIncludes(t *testing.T) { + t.Parallel() + data := []struct { + inc, exc, expected []string + }{ + {nil, nil, nil}, + {[]string{}, []string{}, nil}, + {nil, []string{"*.foo"}, []string{"*.foo"}}, + {[]string{"*.bar"}, []string{"a*.bar"}, []string{"*", "!*.bar", "a*.bar"}}, + {[]string{"*.bar"}, nil, []string{"*", "!*.bar"}}, + } + + for _, d := range data { + assert.EqualValues(t, d.expected, processIncludes(d.inc, d.exc)) + } +} + +func TestPickConfigFile(t *testing.T) { + cmd := &cobra.Command{} + cmd.Flags().String("config", defaultConfigFile, "foo") + + cf, req := pickConfigFile(cmd) + assert.False(t, req) + assert.Equal(t, defaultConfigFile, cf) + + os.Setenv("GOMPLATE_CONFIG", "foo.yaml") + defer os.Unsetenv("GOMPLATE_CONFIG") + cf, req = pickConfigFile(cmd) + assert.True(t, req) + assert.Equal(t, "foo.yaml", cf) + + cmd.ParseFlags([]string{"--config", "config.file"}) + cf, req = pickConfigFile(cmd) + assert.True(t, req) + assert.Equal(t, "config.file", cf) + + os.Setenv("GOMPLATE_CONFIG", "ignored.yaml") + cf, req = pickConfigFile(cmd) + assert.True(t, req) + assert.Equal(t, "config.file", cf) +} + +func TestApplyEnvVars_PluginTimeout(t *testing.T) { + os.Setenv("GOMPLATE_PLUGIN_TIMEOUT", "bogus") + + ctx := context.TODO() + cfg := &config.Config{} + _, err := applyEnvVars(ctx, cfg) + assert.Error(t, err) + + cfg = &config.Config{ + PluginTimeout: 2 * time.Second, + } + expected := &config.Config{ + PluginTimeout: 2 * time.Second, + } + actual, err := applyEnvVars(ctx, cfg) + assert.NoError(t, err) + assert.EqualValues(t, expected, actual) + + os.Setenv("GOMPLATE_PLUGIN_TIMEOUT", "2s") + defer os.Unsetenv("GOMPLATE_PLUGIN_TIMEOUT") + + cfg = &config.Config{} + actual, err = applyEnvVars(ctx, cfg) + assert.NoError(t, err) + assert.EqualValues(t, expected, actual) + + cfg = &config.Config{ + PluginTimeout: 100 * time.Millisecond, + } + expected = &config.Config{ + PluginTimeout: 100 * time.Millisecond, + } + actual, err = applyEnvVars(ctx, cfg) + assert.NoError(t, err) + assert.EqualValues(t, expected, actual) + +} + +func TestApplyEnvVars_SuppressEmpty(t *testing.T) { + os.Setenv("GOMPLATE_SUPPRESS_EMPTY", "bogus") + defer os.Unsetenv("GOMPLATE_SUPPRESS_EMPTY") + + ctx := context.TODO() + cfg := &config.Config{} + expected := &config.Config{ + SuppressEmpty: false, + } + actual, err := applyEnvVars(ctx, cfg) + assert.NoError(t, err) + assert.EqualValues(t, expected, actual) + + os.Setenv("GOMPLATE_SUPPRESS_EMPTY", "true") + + cfg = &config.Config{} + expected = &config.Config{ + SuppressEmpty: true, + } + actual, err = applyEnvVars(ctx, cfg) + assert.NoError(t, err) + assert.EqualValues(t, expected, actual) + + os.Setenv("GOMPLATE_SUPPRESS_EMPTY", "false") + + cfg = &config.Config{ + SuppressEmpty: true, + } + expected = &config.Config{ + SuppressEmpty: true, + } + actual, err = applyEnvVars(ctx, cfg) + assert.NoError(t, err) + assert.EqualValues(t, expected, actual) +} diff --git a/cmd/gomplate/logger.go b/cmd/gomplate/logger.go index d40281a5f..c58dc31dc 100644 --- a/cmd/gomplate/logger.go +++ b/cmd/gomplate/logger.go @@ -12,7 +12,8 @@ import ( ) func initLogger(ctx context.Context) context.Context { - zerolog.SetGlobalLevel(zerolog.InfoLevel) + // default to warn level + zerolog.SetGlobalLevel(zerolog.WarnLevel) zerolog.DurationFieldUnit = time.Second stdlogger := log.With().Bool("stdlog", true).Logger() diff --git a/cmd/gomplate/main.go b/cmd/gomplate/main.go index 62c85fdbf..110a05035 100644 --- a/cmd/gomplate/main.go +++ b/cmd/gomplate/main.go @@ -5,33 +5,26 @@ The gomplate command package main import ( - "bytes" "context" + "fmt" "os" "os/exec" "os/signal" "github.com/hairyhenderson/gomplate/v3" "github.com/hairyhenderson/gomplate/v3/env" + "github.com/hairyhenderson/gomplate/v3/internal/config" "github.com/hairyhenderson/gomplate/v3/version" "github.com/rs/zerolog" - "github.com/spf13/cobra" -) -var ( - verbose bool - execPipe bool - opts gomplate.Config - includes []string - - postRunInput *bytes.Buffer + "github.com/spf13/cobra" ) // postRunExec - if templating succeeds, the command following a '--' will be executed -func postRunExec(cmd *cobra.Command, args []string) error { +func postRunExec(ctx context.Context, cfg *config.Config) error { + args := cfg.PostExec if len(args) > 0 { - ctx := cmd.Context() log := zerolog.Ctx(ctx) log.Debug().Strs("args", args).Msg("running post-exec command") @@ -39,11 +32,7 @@ func postRunExec(cmd *cobra.Command, args []string) error { args = args[1:] // nolint: gosec c := exec.CommandContext(ctx, name, args...) - if execPipe { - c.Stdin = postRunInput - } else { - c.Stdin = os.Stdin - } + c.Stdin = cfg.PostExecInput c.Stderr = os.Stderr c.Stdout = os.Stdout @@ -73,28 +62,10 @@ func optionalExecArgs(cmd *cobra.Command, args []string) error { return cobra.NoArgs(cmd, args) } -// process --include flags - these are analogous to specifying --exclude '*', -// then the inverse of the --include options. -func processIncludes(includes, excludes []string) []string { - out := []string{} - // if any --includes are set, we start by excluding everything - if len(includes) > 0 { - out = make([]string, 1+len(includes)) - out[0] = "*" - } - for i, include := range includes { - // includes are just the opposite of an exclude - out[i+1] = "!" + include - } - out = append(out, excludes...) - return out -} - func newGomplateCmd() *cobra.Command { rootCmd := &cobra.Command{ Use: "gomplate", Short: "Process text files with Go templates", - PreRunE: validateOpts, Version: version.Version, RunE: func(cmd *cobra.Command, args []string) error { if v, _ := cmd.Flags().GetBool("verbose"); v { @@ -103,27 +74,33 @@ func newGomplateCmd() *cobra.Command { ctx := cmd.Context() log := zerolog.Ctx(ctx) - log.Debug().Msgf("%s version %s, build %s\nconfig is:\n%s", - cmd.Name(), version.Version, version.GitCommit, - &opts) + cfg, err := loadConfig(cmd, args) + if err != nil { + return err + } - // support --include - opts.ExcludeGlob = processIncludes(includes, opts.ExcludeGlob) + log.Debug().Msgf("starting %s", cmd.Name()) + log.Debug(). + Str("version", version.Version). + Str("build", version.GitCommit). + Msgf("config is:\n%v", cfg) - if execPipe { - postRunInput = &bytes.Buffer{} - opts.Out = postRunInput - } - err := gomplate.RunTemplates(&opts) + err = gomplate.RunTemplatesWithContext(ctx, cfg) cmd.SilenceErrors = true cmd.SilenceUsage = true - log.Debug().Msgf("rendered %d template(s) with %d error(s) in %v", - gomplate.Metrics.TemplatesProcessed, gomplate.Metrics.Errors, gomplate.Metrics.TotalRenderDuration) - return err + fmt.Fprintf(os.Stderr, "\n") + log.Debug().Int("templatesRendered", gomplate.Metrics.TemplatesProcessed). + Int("errors", gomplate.Metrics.Errors). + Dur("duration", gomplate.Metrics.TotalRenderDuration). + Msg("completed rendering") + + if err != nil { + return err + } + return postRunExec(ctx, cfg) }, - PostRunE: postRunExec, - Args: optionalExecArgs, + Args: optionalExecArgs, } return rootCmd } @@ -131,34 +108,36 @@ func newGomplateCmd() *cobra.Command { func initFlags(command *cobra.Command) { command.Flags().SortFlags = false - command.Flags().StringArrayVarP(&opts.DataSources, "datasource", "d", nil, "`datasource` in alias=URL form. Specify multiple times to add multiple sources.") - command.Flags().StringArrayVarP(&opts.DataSourceHeaders, "datasource-header", "H", nil, "HTTP `header` field in 'alias=Name: value' form to be provided on HTTP-based data sources. Multiples can be set.") + command.Flags().StringSliceP("datasource", "d", nil, "`datasource` in alias=URL form. Specify multiple times to add multiple sources.") + command.Flags().StringSliceP("datasource-header", "H", nil, "HTTP `header` field in 'alias=Name: value' form to be provided on HTTP-based data sources. Multiples can be set.") - command.Flags().StringArrayVarP(&opts.Contexts, "context", "c", nil, "pre-load a `datasource` into the context, in alias=URL form. Use the special alias `.` to set the root context.") + command.Flags().StringSliceP("context", "c", nil, "pre-load a `datasource` into the context, in alias=URL form. Use the special alias `.` to set the root context.") - command.Flags().StringArrayVar(&opts.Plugins, "plugin", nil, "plug in an external command as a function in name=path form. Can be specified multiple times") + command.Flags().StringSlice("plugin", nil, "plug in an external command as a function in name=path form. Can be specified multiple times") - command.Flags().StringArrayVarP(&opts.InputFiles, "file", "f", []string{"-"}, "Template `file` to process. Omit to use standard input, or use --in or --input-dir") - command.Flags().StringVarP(&opts.Input, "in", "i", "", "Template `string` to process (alternative to --file and --input-dir)") - command.Flags().StringVar(&opts.InputDir, "input-dir", "", "`directory` which is examined recursively for templates (alternative to --file and --in)") + command.Flags().StringSliceP("file", "f", []string{"-"}, "Template `file` to process. Omit to use standard input, or use --in or --input-dir") + command.Flags().StringP("in", "i", "", "Template `string` to process (alternative to --file and --input-dir)") + command.Flags().String("input-dir", "", "`directory` which is examined recursively for templates (alternative to --file and --in)") - command.Flags().StringArrayVar(&opts.ExcludeGlob, "exclude", []string{}, "glob of files to not parse") - command.Flags().StringArrayVar(&includes, "include", []string{}, "glob of files to parse") + command.Flags().StringSlice("exclude", []string{}, "glob of files to not parse") + command.Flags().StringSlice("include", []string{}, "glob of files to parse") - command.Flags().StringArrayVarP(&opts.OutputFiles, "out", "o", []string{"-"}, "output `file` name. Omit to use standard output.") - command.Flags().StringArrayVarP(&opts.Templates, "template", "t", []string{}, "Additional template file(s)") - command.Flags().StringVar(&opts.OutputDir, "output-dir", ".", "`directory` to store the processed templates. Only used for --input-dir") - command.Flags().StringVar(&opts.OutputMap, "output-map", "", "Template `string` to map the input file to an output path") - command.Flags().StringVar(&opts.OutMode, "chmod", "", "set the mode for output file(s). Omit to inherit from input file(s)") + command.Flags().StringSliceP("out", "o", []string{"-"}, "output `file` name. Omit to use standard output.") + command.Flags().StringSliceP("template", "t", []string{}, "Additional template file(s)") + command.Flags().String("output-dir", ".", "`directory` to store the processed templates. Only used for --input-dir") + command.Flags().String("output-map", "", "Template `string` to map the input file to an output path") + command.Flags().String("chmod", "", "set the mode for output file(s). Omit to inherit from input file(s)") - command.Flags().BoolVar(&execPipe, "exec-pipe", false, "pipe the output to the post-run exec command") + command.Flags().Bool("exec-pipe", false, "pipe the output to the post-run exec command") ldDefault := env.Getenv("GOMPLATE_LEFT_DELIM", "{{") rdDefault := env.Getenv("GOMPLATE_RIGHT_DELIM", "}}") - command.Flags().StringVar(&opts.LDelim, "left-delim", ldDefault, "override the default left-`delimiter` [$GOMPLATE_LEFT_DELIM]") - command.Flags().StringVar(&opts.RDelim, "right-delim", rdDefault, "override the default right-`delimiter` [$GOMPLATE_RIGHT_DELIM]") + command.Flags().String("left-delim", ldDefault, "override the default left-`delimiter` [$GOMPLATE_LEFT_DELIM]") + command.Flags().String("right-delim", rdDefault, "override the default right-`delimiter` [$GOMPLATE_RIGHT_DELIM]") + + command.Flags().BoolP("verbose", "V", false, "output extra information about what gomplate is doing") - command.Flags().BoolVarP(&verbose, "verbose", "V", false, "output extra information about what gomplate is doing") + command.Flags().String("config", defaultConfigFile, "config file (overridden by commandline flags)") } func main() { diff --git a/cmd/gomplate/main_test.go b/cmd/gomplate/main_test.go index e0d74e9d3..06ab7d0f9 100644 --- a/cmd/gomplate/main_test.go +++ b/cmd/gomplate/main_test.go @@ -1,23 +1 @@ package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestProcessIncludes(t *testing.T) { - data := []struct { - inc, exc, expected []string - }{ - {nil, nil, []string{}}, - {[]string{}, []string{}, []string{}}, - {nil, []string{"*.foo"}, []string{"*.foo"}}, - {[]string{"*.bar"}, []string{"a*.bar"}, []string{"*", "!*.bar", "a*.bar"}}, - {[]string{"*.bar"}, nil, []string{"*", "!*.bar"}}, - } - - for _, d := range data { - assert.EqualValues(t, d.expected, processIncludes(d.inc, d.exc)) - } -} diff --git a/cmd/gomplate/validate.go b/cmd/gomplate/validate.go deleted file mode 100644 index f4c73133f..000000000 --- a/cmd/gomplate/validate.go +++ /dev/null @@ -1,63 +0,0 @@ -package main - -import ( - "fmt" - "strings" - - "github.com/spf13/cobra" -) - -func notTogether(cmd *cobra.Command, flags ...string) error { - found := "" - for _, flag := range flags { - f := cmd.Flag(flag) - if f != nil && f.Changed { - if found != "" { - a := make([]string, len(flags)) - for i := range a { - a[i] = "--" + flags[i] - } - return fmt.Errorf("only one of these flags is supported at a time: %s", strings.Join(a, ", ")) - } - found = flag - } - } - return nil -} - -func mustTogether(cmd *cobra.Command, left, right string) error { - l := cmd.Flag(left) - if l != nil && l.Changed { - r := cmd.Flag(right) - if r != nil && !r.Changed { - return fmt.Errorf("--%s must be set when --%s is set", right, left) - } - } - - return nil -} - -func validateOpts(cmd *cobra.Command, args []string) (err error) { - err = notTogether(cmd, "in", "file", "input-dir") - if err == nil { - err = notTogether(cmd, "out", "output-dir", "output-map", "exec-pipe") - } - - if err == nil && len(opts.InputFiles) != len(opts.OutputFiles) { - err = fmt.Errorf("must provide same number of --out (%d) as --file (%d) options", len(opts.OutputFiles), len(opts.InputFiles)) - } - - if err == nil && cmd.Flag("exec-pipe").Changed && len(args) == 0 { - err = fmt.Errorf("--exec-pipe may only be used with a post-exec command after --") - } - - if err == nil { - err = mustTogether(cmd, "output-dir", "input-dir") - } - - if err == nil { - err = mustTogether(cmd, "output-map", "input-dir") - } - - return err -} diff --git a/cmd/gomplate/validate_test.go b/cmd/gomplate/validate_test.go deleted file mode 100644 index 54f4ba017..000000000 --- a/cmd/gomplate/validate_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package main - -import ( - "testing" - - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" -) - -func TestValidateOpts(t *testing.T) { - err := validateOpts(parseFlags()) - assert.NoError(t, err) - - err = validateOpts(parseFlags("-i=foo", "-f", "bar")) - assert.Error(t, err) - - err = validateOpts(parseFlags("-i=foo", "-o=bar", "-o=baz")) - assert.Error(t, err) - - err = validateOpts(parseFlags("-i=foo", "--input-dir=baz")) - assert.Error(t, err) - - err = validateOpts(parseFlags("--input-dir=foo", "-f=bar")) - assert.Error(t, err) - - err = validateOpts(parseFlags("--output-dir=foo", "-o=bar")) - assert.Error(t, err) - - err = validateOpts(parseFlags("--output-dir=foo")) - assert.Error(t, err) - - err = validateOpts(parseFlags("--output-map", "bar")) - assert.Error(t, err) - - err = validateOpts(parseFlags("-o", "foo", "--output-map", "bar")) - assert.Error(t, err) - - err = validateOpts(parseFlags( - "--input-dir", "in", - "--output-dir", "foo", - "--output-map", "bar", - )) - assert.Error(t, err) - - err = validateOpts(parseFlags("--exec-pipe")) - assert.Error(t, err) - - err = validateOpts(parseFlags("--exec-pipe", "--")) - assert.Error(t, err) - - err = validateOpts(parseFlags( - "--exec-pipe", - "--", "echo", "foo", - )) - assert.NoError(t, err) - - err = validateOpts(parseFlags( - "--exec-pipe", - "--out", "foo", - "--", "echo", - )) - assert.Error(t, err) - - err = validateOpts(parseFlags( - "--input-dir", "in", - "--exec-pipe", - "--output-dir", "foo", - "--", "echo", - )) - assert.Error(t, err) - - err = validateOpts(parseFlags( - "--input-dir", "in", - "--exec-pipe", - "--output-map", "foo", - "--", "echo", - )) - assert.Error(t, err) - - err = validateOpts(parseFlags( - "--input-dir", "in", - "--output-map", "bar", - )) - assert.NoError(t, err) -} - -func parseFlags(flags ...string) (cmd *cobra.Command, args []string) { - cmd = &cobra.Command{} - initFlags(cmd) - err := cmd.ParseFlags(flags) - if err != nil { - panic(err) - } - return cmd, cmd.Flags().Args() -} diff --git a/config.go b/config.go index 18bd5546a..913370a50 100644 --- a/config.go +++ b/config.go @@ -2,9 +2,9 @@ package gomplate import ( "io" - "os" - "strconv" "strings" + + "github.com/hairyhenderson/gomplate/v3/internal/config" ) // Config - values necessary for rendering templates with gomplate. @@ -52,20 +52,6 @@ func (o *Config) defaults() *Config { return o } -// parse an os.FileMode out of the string, and let us know if it's an override or not... -func (o *Config) getMode() (os.FileMode, bool, error) { - modeOverride := o.OutMode != "" - m, err := strconv.ParseUint("0"+o.OutMode, 8, 32) - if err != nil { - return 0, false, err - } - mode := os.FileMode(m) - if mode == 0 && o.Input != "" { - mode = 0644 - } - return mode, modeOverride, nil -} - // nolint: gocyclo func (o *Config) String() string { o.defaults() @@ -124,3 +110,29 @@ func (o *Config) String() string { } return c } + +func (o *Config) toNewConfig() (*config.Config, error) { + cfg := &config.Config{ + Input: o.Input, + InputFiles: o.InputFiles, + InputDir: o.InputDir, + ExcludeGlob: o.ExcludeGlob, + OutputFiles: o.OutputFiles, + OutputDir: o.OutputDir, + OutputMap: o.OutputMap, + OutMode: o.OutMode, + LDelim: o.LDelim, + RDelim: o.RDelim, + Templates: o.Templates, + OutWriter: o.Out, + } + err := cfg.ParsePluginFlags(o.Plugins) + if err != nil { + return nil, err + } + err = cfg.ParseDataSourceFlags(o.DataSources, o.Contexts, o.DataSourceHeaders) + if err != nil { + return nil, err + } + return cfg, nil +} diff --git a/config_test.go b/config_test.go index 91c2b1a9f..0ddefc3e8 100644 --- a/config_test.go +++ b/config_test.go @@ -1,7 +1,6 @@ package gomplate import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -47,27 +46,3 @@ output: {{ .in }}` assert.Equal(t, expected, c.String()) } - -func TestGetMode(t *testing.T) { - c := &Config{} - m, o, err := c.getMode() - assert.NoError(t, err) - assert.Equal(t, os.FileMode(0), m) - assert.False(t, o) - - c = &Config{OutMode: "755"} - m, o, err = c.getMode() - assert.NoError(t, err) - assert.Equal(t, os.FileMode(0755), m) - assert.True(t, o) - - c = &Config{OutMode: "0755"} - m, o, err = c.getMode() - assert.NoError(t, err) - assert.Equal(t, os.FileMode(0755), m) - assert.True(t, o) - - c = &Config{OutMode: "foo"} - _, _, err = c.getMode() - assert.Error(t, err) -} diff --git a/context.go b/context.go index ab12d404b..a2b8b0026 100644 --- a/context.go +++ b/context.go @@ -1,10 +1,12 @@ package gomplate import ( + "context" "os" "strings" "github.com/hairyhenderson/gomplate/v3/data" + "github.com/hairyhenderson/gomplate/v3/internal/config" ) // context for templates @@ -20,11 +22,10 @@ func (c *tmplctx) Env() map[string]string { return env } -func createTmplContext(contexts []string, d *data.Data) (interface{}, error) { +func createTmplContext(ctx context.Context, contexts config.DSources, d *data.Data) (interface{}, error) { var err error tctx := &tmplctx{} - for _, c := range contexts { - a := parseAlias(c) + for a := range contexts { if a == "." { return d.Datasource(a) } @@ -35,13 +36,3 @@ func createTmplContext(contexts []string, d *data.Data) (interface{}, error) { } return tctx, nil } - -func parseAlias(arg string) string { - parts := strings.SplitN(arg, "=", 2) - switch len(parts) { - case 1: - return strings.SplitN(parts[0], ".", 2)[0] - default: - return parts[0] - } -} diff --git a/context_test.go b/context_test.go index ab2737497..94f2b51b4 100644 --- a/context_test.go +++ b/context_test.go @@ -1,11 +1,13 @@ package gomplate import ( + "context" "net/url" "os" "testing" "github.com/hairyhenderson/gomplate/v3/data" + "github.com/hairyhenderson/gomplate/v3/internal/config" "github.com/stretchr/testify/assert" ) @@ -24,7 +26,8 @@ func TestEnvGetsUpdatedEnvironment(t *testing.T) { } func TestCreateContext(t *testing.T) { - c, err := createTmplContext(nil, nil) + ctx := context.TODO() + c, err := createTmplContext(ctx, nil, nil) assert.NoError(t, err) assert.Empty(t, c) @@ -40,31 +43,18 @@ func TestCreateContext(t *testing.T) { } os.Setenv("foo", "foo: bar") defer os.Unsetenv("foo") - c, err = createTmplContext([]string{"foo=" + fooURL}, d) + c, err = createTmplContext(ctx, map[string]config.DSConfig{"foo": {URL: uf}}, d) assert.NoError(t, err) assert.IsType(t, &tmplctx{}, c) - ctx := c.(*tmplctx) - ds := ((*ctx)["foo"]).(map[string]interface{}) + tctx := c.(*tmplctx) + ds := ((*tctx)["foo"]).(map[string]interface{}) assert.Equal(t, "bar", ds["foo"]) os.Setenv("bar", "bar: baz") defer os.Unsetenv("bar") - c, err = createTmplContext([]string{".=" + barURL}, d) + c, err = createTmplContext(ctx, map[string]config.DSConfig{".": {URL: ub}}, d) assert.NoError(t, err) assert.IsType(t, map[string]interface{}{}, c) ds = c.(map[string]interface{}) assert.Equal(t, "baz", ds["bar"]) } - -func TestParseAlias(t *testing.T) { - testdata := map[string]string{ - "": "", - "foo": "foo", - "foo.bar": "foo", - "a=b": "a", - ".=foo": ".", - } - for k, v := range testdata { - assert.Equal(t, v, parseAlias(k)) - } -} diff --git a/data/datasource.go b/data/datasource.go index 1fcccf98f..456b5a12f 100644 --- a/data/datasource.go +++ b/data/datasource.go @@ -16,6 +16,7 @@ import ( "github.com/pkg/errors" + "github.com/hairyhenderson/gomplate/v3/internal/config" "github.com/hairyhenderson/gomplate/v3/libkv" "github.com/hairyhenderson/gomplate/v3/vault" ) @@ -125,6 +126,29 @@ func NewData(datasourceArgs, headerArgs []string) (*Data, error) { return data, nil } +// FromConfig - internal use only! +func FromConfig(cfg *config.Config) *Data { + sources := map[string]*Source{} + for alias, d := range cfg.DataSources { + sources[alias] = &Source{ + Alias: alias, + URL: d.URL, + header: d.Header, + } + } + for alias, d := range cfg.Context { + sources[alias] = &Source{ + Alias: alias, + URL: d.URL, + header: d.Header, + } + } + return &Data{ + Sources: sources, + extraHeaders: cfg.ExtraHeaders, + } +} + // Source - a data source type Source struct { Alias string diff --git a/data/datasource_test.go b/data/datasource_test.go index cdef39451..6752d9e5e 100644 --- a/data/datasource_test.go +++ b/data/datasource_test.go @@ -2,6 +2,7 @@ package data import ( "fmt" + "net/http" "net/url" "os" "path/filepath" @@ -9,6 +10,7 @@ import ( "strings" "testing" + "github.com/hairyhenderson/gomplate/v3/internal/config" "github.com/spf13/afero" "github.com/stretchr/testify/assert" @@ -455,3 +457,70 @@ func TestAbsFileURL(t *testing.T) { assert.NoError(t, err) assert.EqualValues(t, expected, u) } + +func TestFromConfig(t *testing.T) { + cfg := &config.Config{} + expected := &Data{ + Sources: map[string]*Source{}, + } + assert.EqualValues(t, expected, FromConfig(cfg)) + + cfg = &config.Config{ + DataSources: map[string]config.DSConfig{ + "foo": { + URL: mustParseURL("http://example.com"), + }, + }, + } + expected = &Data{ + Sources: map[string]*Source{ + "foo": { + Alias: "foo", + URL: mustParseURL("http://example.com"), + }, + }, + } + assert.EqualValues(t, expected, FromConfig(cfg)) + + cfg = &config.Config{ + DataSources: map[string]config.DSConfig{ + "foo": { + URL: mustParseURL("http://foo.com"), + }, + }, + Context: map[string]config.DSConfig{ + "bar": { + URL: mustParseURL("http://bar.com"), + Header: http.Header{ + "Foo": []string{"bar"}, + }, + }, + }, + ExtraHeaders: map[string]http.Header{ + "baz": { + "Foo": []string{"bar"}, + }, + }, + } + expected = &Data{ + Sources: map[string]*Source{ + "foo": { + Alias: "foo", + URL: mustParseURL("http://foo.com"), + }, + "bar": { + Alias: "bar", + URL: mustParseURL("http://bar.com"), + header: http.Header{ + "Foo": []string{"bar"}, + }, + }, + }, + extraHeaders: map[string]http.Header{ + "baz": { + "Foo": []string{"bar"}, + }, + }, + } + assert.EqualValues(t, expected, FromConfig(cfg)) +} diff --git a/docs/content/config.md b/docs/content/config.md new file mode 100644 index 000000000..7857e4524 --- /dev/null +++ b/docs/content/config.md @@ -0,0 +1,352 @@ +--- +title: Configuration +weight: 12 +menu: main +--- + +In addition to [command-line arguments][], gomplate supports the use of +configuration files to control its behaviour. + +Using a file for configuration can be useful especially when rendering templates +that use multiple datasources, plugins, nested templates, etc... In situations +where teams share templates, it can be helpful to commit config files into the +team's source control system. + +By default, gomplate will look for a file `.gomplate.yaml` in the current working +diretory, but this path can be altered with the [`--config`](../usage/#--config) +command-line argument, or the `GOMPLATE_CONFIG` environment variable. + +### Configuration precedence + +[Command-line arguments][] will always take precedence over settings in a config +file. In the cases where configuration can be altered with an environment +variable, the config file will take precedence over environment variables. + +So, if the `leftDelim` setting is configured in 3 ways: + +```console +$ export GOMPLATE_LEFT_DELIM=:: +$ echo "leftDelim: ((" > .gomplate.yaml +$ gomplate --left-delim "<<" +``` + +The delimiter will be `<<`. + +## File format + +Currently, gomplate supports config files written in [YAML][] syntax, though other +structured formats may be supported in future (please [file an issue][] if this +is important to you!) + +Roughly all of the [command-line arguments][] are able to be set in a config +file, with the exception of `--help`, `--verbose`, and `--version`. Some +environment variable based settings not configurable on the command-line are +also supported in config files. + +Most of the configuration names are similar, though instead of using `kebab-case`, +multi-word names are rendered as `camelCase`. + +Here is an example of a simple config file: + +```yaml +inputDir: in/ +outputDir: out/ + +datasources: + local: + url: file:///tmp/data.json + remote: + url: https://example.com/api/v1/data + header: + Authorization: ["Basic aGF4MHI6c3dvcmRmaXNoCg=="] + +plugins: + dostuff: /usr/local/bin/stuff.sh +``` + +## `chmod` + +See [`--chmod`](../usage/#--chmod). + +Sets the output file mode. + +## `context` + +See [`--context`](../usage/#--context-c). + +Add data sources to the default context. This is a nested structure that +includes the URL for the data source and the optional HTTP header to send. + +For example: + +```yaml +context: + data: + url: https://example.com/api/v1/data + header: + Authorization: ["Basic aGF4MHI6c3dvcmRmaXNoCg=="] + stuff: + url: stuff.yaml +``` + +This adds two datasources to the context: `data` and `stuff`, and when the `data` +source is retrieved, an `Authorization` header will be sent with the given value. + +Note that the `.` name can also be used to set the entire context: + +```yaml +context: + .: + url: data.toml +``` + +## `datasources` + +See [`--datasource`](../usage/#--datasource-d). + +Define data sources. This is a nested structure that includes the URL for the data +source and the optional HTTP header to send. + +For example: + +```yaml +datasources: + data: + url: https://example.com/api/v1/data + header: + Authorization: ["Basic aGF4MHI6c3dvcmRmaXNoCg=="] + stuff: + url: stuff.yaml +``` + +This defines two datasources: `data` and `stuff`, and when the `data` +source is used, an `Authorization` header will be sent with the given value. + +## `excludes` + +See [`--exclude` and `--include`](../usage/#--exclude-and---include). + +This is an array of exclude patterns, used in conjunction with [`inputDir`](#inputdir). +Note that there is no `includes`, instead you can specify negative +exclusions by prefixing the patterns with `!`. + +```yaml +excludes: + - '*.txt' + - '!include-this.txt' +``` + +This will skip all files with the extension `.txt`, except for files named +`include-this.txt`, which will be processed. + +## `execPipe` + +See [`--exec-pipe`](../usage/#--exec-pipe). + +Use the rendered output as the [`postExec`](#postexec) command's standard input. + +Must be used in conjuction with [`postExec`](#postexec), and will override +any [`outputFiles`](#outputfiles) settings. + +## `in` + +See [`--in`/`-i`](../usage/#--file-f---in-i-and---out-o). + +Provide the input template inline. Note that unlike the `--in`/`-i` commandline +argument, there are no shell-imposed length limits. + +A simple example: +```yaml +in: hello to {{ .Env.USER }} +``` + +A multi-line example (see https://yaml-multiline.info/ for more about multi-line +string syntax in YAML): + +```yaml +in: | + A longer multi-line + document: + {{- range .foo }} + {{ .bar }} + {{ end }} +``` + +May not be used with `inputDir` or `inputFiles`. + +## `inputDir` + +See [`--input-dir`](../usage/#--input-dir-and---output-dir). + +The directory containing input template files. Must be used with +[`outputDir`](#outputdir) or [`outputMap`](#outputmap). Can also be used with [`excludes`](#excludes). + +```yaml +inputDir: templates/ +outputDir: out/ +``` + +May not be used with `in` or `inputFiles`. + +## `inputFiles` + +See [`--file`/`-f`](../usage/#--file-f---in-i-and---out-o). + +An array of input template paths. The special value `-` means `Stdin`. Multiple +values can be set, but there must be a corresponding number of `outputFiles` +entries present. + +```yaml +inputFiles: + - first.tmpl + - second.tmpl +outputFiles: + - first.out + - second.out +``` + +Flow style can be more compact: + +```yaml +inputFiles: ['-'] +outputFiles: ['-'] +``` + +May not be used with `in` or `inputDir`. + +## `leftDelim` + +See [`--left-delim`](../usage/#overriding-the-template-delimiters). + +Overrides the left template delimiter. + +```yaml +leftDelim: '%{' +``` + +## `outputDir` + +See [`--output-dir`](../usage/#--input-dir-and---output-dir). + +The directory to write rendered output files. Must be used with +[`inputDir`](#inputdir). + +```yaml +inputDir: templates/ +outputDir: out/ +``` + +May not be used with `outputFiles`. + +## `outputFiles` + +See [`--out`/`-o`](../usage/#--file-f---in-i-and---out-o). + +An array of output file paths. The special value `-` means `Stdout`. Multiple +values can be set, but there must be a corresponding number of `inputFiles` +entries present. + +```yaml +inputFiles: + - first.tmpl + - second.tmpl +outputFiles: + - first.out + - second.out +``` + +Can also be used with [`in`](#in): + +```yaml +in: >- + hello, + world! +outputFiles: [ hello.txt ] +``` + +May not be used with `inputDir`. + +## `outputMap` + +See [`--output-map`](../usage/#--output-map). + +Must be used with [`inputDir`](#inputdir). + +```yaml +inputDir: in/ +outputMap: | + out/{{ .in | strings.ReplaceAll ".yaml.tmpl" ".yaml" }} +``` + +## `plugins` + +See [`--plugin`](../usage/#--plugin). + +An mapping of key/value pairs to plug in custom functions for use in the templates. + +```yaml +in: '{{ "hello world" | figlet | lolcat }}' +plugins: + figlet: /usr/local/bin/figlet + lolcat: /home/hairyhenderson/go/bin/lolcat +``` + +## `pluginTimeout` + +See [`--plugin`](../usage/#--plugin). + +Sets the timeout for running plugins. By default, plugins will time out after 5 +seconds. This value can be set to override this default. The value must be +a valid [duration](../functions/time/#time-parseduration) such as `10s` or `3m`. + +```yaml +plugins: + figlet: /usr/local/bin/figlet +pluginTimeout: 500ms +``` + +## `postExec` + +See [post-template command execution](../usage/#post-template-command-execution). + +Configures a command to run after the template is rendered. + +See also [`execPipe`](#execpipe) for piping output directly into the `postExec` command. + +## `rightDelim` + +See [`--right-delim`](../usage/#overriding-the-template-delimiters). + +Overrides the right template delimiter. + +```yaml +rightDelim: '))' +``` + +## `suppressEmpty` + +See _[Suppressing empty output](../usage/#suppressing-empty-output)_ + +Suppresses empty output (i.e. output consisting of only whitespace). Can also be set with the `GOMPLATE_SUPPRESS_EMPTY` environment variable. + +```yaml +suppressEmpty: true +``` + +## `templates` + +See [`--template`/`-t`](../usage/#--template-t). + +An array of template references. Can be just a path or an alias and a path: + +```yaml +templates: + - t=foo/bar/helloworld.tmpl + - templatedir/ + - dir=foo/bar/ + - mytemplate.t +``` + +[command-line arguments]: ../usage +[file an issue]: https://github.com/hairyhenderson/gomplate/issues/new +[YAML]: http://yaml.org diff --git a/docs/content/datasources.md b/docs/content/datasources.md index 6f9872d56..6bf721723 100644 --- a/docs/content/datasources.md +++ b/docs/content/datasources.md @@ -1,6 +1,6 @@ --- title: Datasources -weight: 13 +weight: 14 menu: main --- diff --git a/docs/content/syntax.md b/docs/content/syntax.md index c7995217f..14304eb32 100644 --- a/docs/content/syntax.md +++ b/docs/content/syntax.md @@ -1,6 +1,6 @@ --- title: Syntax -weight: 12 +weight: 13 menu: main --- diff --git a/docs/content/usage.md b/docs/content/usage.md index e5dc7d92e..375011222 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -19,6 +19,23 @@ Hello, hairyhenderson ## Commandline Arguments +### `--config` + +Specify the path to a [gomplate config file](../config). The default is `.gomplate.yaml`. Can also be set with the `GOMPLATE_CONFIG` environment variable. + +For example: + +```console +$ cat myconfig.yaml +in: hello {{ .data.thing }} + +datasources: + data: + url: https://example.com/data.json +$ gomplate --config myconfig.yaml +hello world +``` + ### `--file`/`-f`, `--in`/`-i`, and `--out`/`-o` By default, `gomplate` will read from `Stdin` and write to `Stdout`. This behaviour can be changed. @@ -135,7 +152,7 @@ A few different forms are valid: - `mydata.json` - This form infers the name from the file name (without extension). Only valid for files in the current directory. -### `--context`/`c` +### `--context`/`-c` Add a data source in `name=URL` form, and make it available in the [default context][] as `.`. The special name `.` (period) can be used to override the entire default context. @@ -262,7 +279,7 @@ post-exec command. ## Suppressing empty output -Sometimes it can be desirable to suppress empty output (i.e. output consisting of only whitespace). To do so, set `GOMPLATE_SUPPRESS_EMPTY=true` in your environment: +Sometimes it can be desirable to suppress empty output (i.e. output consisting of only whitespace). To do so, set `suppressEmpty: true` in your [config][] file, or `GOMPLATE_SUPPRESS_EMPTY=true` in your environment: ```console $ export GOMPLATE_SUPPRESS_EMPTY=true @@ -273,5 +290,6 @@ cat: out: No such file or directory [default context]: ../syntax/#the-context [context]: ../syntax/#the-context +[config]: ../config/#suppressempty [external templates]: ../syntax/#external-templates [`.gitignore`]: https://git-scm.com/docs/gitignore diff --git a/docs/static/images/gomplate-gh.png b/docs/static/images/gomplate-gh.png new file mode 100644 index 000000000..84d49c775 Binary files /dev/null and b/docs/static/images/gomplate-gh.png differ diff --git a/docs/static/images/gomplate-large.png b/docs/static/images/gomplate-large.png new file mode 100644 index 000000000..3e8dd782b Binary files /dev/null and b/docs/static/images/gomplate-large.png differ diff --git a/gomplate.go b/gomplate.go index d796c4dd2..fe59c2ac9 100644 --- a/gomplate.go +++ b/gomplate.go @@ -5,6 +5,7 @@ package gomplate import ( "bytes" "context" + "fmt" "io" "os" "path" @@ -14,7 +15,9 @@ import ( "time" "github.com/hairyhenderson/gomplate/v3/data" + "github.com/hairyhenderson/gomplate/v3/internal/config" "github.com/pkg/errors" + "github.com/rs/zerolog" "github.com/spf13/afero" ) @@ -109,42 +112,45 @@ func parseTemplateArg(templateArg string, ta templateAliases) error { // RunTemplates - run all gomplate templates specified by the given configuration func RunTemplates(o *Config) error { - return RunTemplatesWithContext(context.Background(), o) + cfg, err := o.toNewConfig() + if err != nil { + return err + } + return RunTemplatesWithContext(context.Background(), cfg) } // RunTemplatesWithContext - run all gomplate templates specified by the given configuration -func RunTemplatesWithContext(ctx context.Context, o *Config) error { +func RunTemplatesWithContext(ctx context.Context, cfg *config.Config) error { + log := zerolog.Ctx(ctx) + Metrics = newMetrics() defer runCleanupHooks() - // make sure config is sane - o.defaults() - ds := append(o.DataSources, o.Contexts...) - d, err := data.NewData(ds, o.DataSourceHeaders) - if err != nil { - return err - } + + d := data.FromConfig(cfg) + log.Debug().Str("data", fmt.Sprintf("%+v", d)).Msg("created data from config") + addCleanupHook(d.Cleanup) - nested, err := parseTemplateArgs(o.Templates) + nested, err := parseTemplateArgs(cfg.Templates) if err != nil { return err } - c, err := createTmplContext(o.Contexts, d) + c, err := createTmplContext(ctx, cfg.Context, d) if err != nil { return err } funcMap := Funcs(d) - err = bindPlugins(ctx, o.Plugins, funcMap) + err = bindPlugins(ctx, cfg, funcMap) if err != nil { return err } - g := newGomplate(funcMap, o.LDelim, o.RDelim, nested, c) + g := newGomplate(funcMap, cfg.LDelim, cfg.RDelim, nested, c) - return g.runTemplates(ctx, o) + return g.runTemplates(ctx, cfg) } -func (g *gomplate) runTemplates(ctx context.Context, o *Config) error { +func (g *gomplate) runTemplates(ctx context.Context, cfg *config.Config) error { start := time.Now() - tmpl, err := gatherTemplates(o, chooseNamer(o, g)) + tmpl, err := gatherTemplates(cfg, chooseNamer(cfg, g)) Metrics.GatherDuration = time.Since(start) if err != nil { Metrics.Errors++ @@ -166,11 +172,11 @@ func (g *gomplate) runTemplates(ctx context.Context, o *Config) error { return nil } -func chooseNamer(o *Config, g *gomplate) func(string) (string, error) { - if o.OutputMap == "" { - return simpleNamer(o.OutputDir) +func chooseNamer(cfg *config.Config, g *gomplate) func(string) (string, error) { + if cfg.OutputMap == "" { + return simpleNamer(cfg.OutputDir) } - return mappingNamer(o.OutputMap, g) + return mappingNamer(cfg.OutputMap, g) } func simpleNamer(outDir string) func(inPath string) (string, error) { diff --git a/internal/config/configfile.go b/internal/config/configfile.go new file mode 100644 index 000000000..eb9b713eb --- /dev/null +++ b/internal/config/configfile.go @@ -0,0 +1,538 @@ +package config + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + "gopkg.in/yaml.v3" +) + +var ( + // PluginTimeoutKey - context key for PluginTimeout - temporary! + PluginTimeoutKey = struct{}{} +) + +// Parse a config file +func Parse(in io.Reader) (*Config, error) { + out := &Config{} + dec := yaml.NewDecoder(in) + err := dec.Decode(out) + if err != nil && err != io.EOF { + return out, err + } + return out, nil +} + +// Config - +type Config struct { + Input string `yaml:"in,omitempty"` + InputFiles []string `yaml:"inputFiles,omitempty,flow"` + InputDir string `yaml:"inputDir,omitempty"` + ExcludeGlob []string `yaml:"excludes,omitempty"` + OutputFiles []string `yaml:"outputFiles,omitempty,flow"` + OutputDir string `yaml:"outputDir,omitempty"` + OutputMap string `yaml:"outputMap,omitempty"` + + SuppressEmpty bool `yaml:"suppressEmpty,omitempty"` + ExecPipe bool `yaml:"execPipe,omitempty"` + PostExec []string `yaml:"postExec,omitempty,flow"` + + OutMode string `yaml:"chmod,omitempty"` + LDelim string `yaml:"leftDelim,omitempty"` + RDelim string `yaml:"rightDelim,omitempty"` + DataSources DSources `yaml:"datasources,omitempty"` + Context DSources `yaml:"context,omitempty"` + Plugins map[string]string `yaml:"plugins,omitempty"` + PluginTimeout time.Duration `yaml:"pluginTimeout,omitempty"` + Templates []string `yaml:"templates,omitempty"` + + // Extra HTTP headers not attached to pre-defined datsources. Potentially + // used by datasources defined in the template. + ExtraHeaders map[string]http.Header `yaml:"-"` + + // internal use only, can't be injected in YAML + PostExecInput io.ReadWriter `yaml:"-"` + OutWriter io.Writer `yaml:"-"` +} + +// DSources - map of datasource configs +type DSources map[string]DSConfig + +func (d DSources) mergeFrom(o DSources) DSources { + for k, v := range o { + c, ok := d[k] + if ok { + d[k] = c.mergeFrom(v) + } else { + d[k] = v + } + } + return d +} + +// DSConfig - datasource config +type DSConfig struct { + URL *url.URL `yaml:"-"` + Header http.Header `yaml:"header,omitempty,flow"` +} + +// UnmarshalYAML - satisfy the yaml.Umarshaler interface - URLs aren't +// well supported, and anyway we need to do some extra parsing +func (d *DSConfig) UnmarshalYAML(value *yaml.Node) error { + type raw struct { + URL string + Header http.Header + } + r := raw{} + err := value.Decode(&r) + if err != nil { + return err + } + u, err := parseSourceURL(r.URL) + if err != nil { + return fmt.Errorf("could not parse datasource URL %q: %w", r.URL, err) + } + *d = DSConfig{ + URL: u, + Header: r.Header, + } + return nil +} + +// MarshalYAML - satisfy the yaml.Marshaler interface - URLs aren't +// well supported, and anyway we need to do some extra parsing +func (d DSConfig) MarshalYAML() (interface{}, error) { + type raw struct { + URL string + Header http.Header + } + r := raw{ + URL: d.URL.String(), + Header: d.Header, + } + return r, nil +} + +func (d DSConfig) mergeFrom(o DSConfig) DSConfig { + if o.URL != nil { + d.URL = o.URL + } + if d.Header == nil { + d.Header = o.Header + } else { + for k, v := range o.Header { + d.Header[k] = v + } + } + return d +} + +// MergeFrom - use this Config as the defaults, and override it with any +// non-zero values from the other Config +// +// Note that Input/InputDir/InputFiles will override each other, as well as +// OutputDir/OutputFiles. +func (c *Config) MergeFrom(o *Config) *Config { + switch { + case !isZero(o.Input): + c.Input = o.Input + c.InputDir = "" + c.InputFiles = nil + c.OutputDir = "" + case !isZero(o.InputDir): + c.Input = "" + c.InputDir = o.InputDir + c.InputFiles = nil + case !isZero(o.InputFiles): + if !(len(o.InputFiles) == 1 && o.InputFiles[0] == "-") { + c.Input = "" + c.InputFiles = o.InputFiles + c.InputDir = "" + c.OutputDir = "" + } + } + + if !isZero(o.OutputMap) { + c.OutputDir = "" + c.OutputFiles = nil + c.OutputMap = o.OutputMap + } + if !isZero(o.OutputDir) { + c.OutputDir = o.OutputDir + c.OutputFiles = nil + c.OutputMap = "" + } + if !isZero(o.OutputFiles) { + c.OutputDir = "" + c.OutputFiles = o.OutputFiles + c.OutputMap = "" + } + if !isZero(o.ExecPipe) { + c.ExecPipe = o.ExecPipe + c.PostExec = o.PostExec + c.OutputFiles = o.OutputFiles + } + if !isZero(o.ExcludeGlob) { + c.ExcludeGlob = o.ExcludeGlob + } + if !isZero(o.OutMode) { + c.OutMode = o.OutMode + } + if !isZero(o.LDelim) { + c.LDelim = o.LDelim + } + if !isZero(o.RDelim) { + c.RDelim = o.RDelim + } + if !isZero(o.Templates) { + c.Templates = o.Templates + } + c.DataSources.mergeFrom(o.DataSources) + c.Context.mergeFrom(o.Context) + if len(o.Plugins) > 0 { + for k, v := range o.Plugins { + c.Plugins[k] = v + } + } + + return c +} + +// ParseDataSourceFlags - sets the DataSources and Context fields from the +// key=value format flags as provided at the command-line +func (c *Config) ParseDataSourceFlags(datasources, contexts, headers []string) error { + for _, d := range datasources { + k, ds, err := parseDatasourceArg(d) + if err != nil { + return err + } + if c.DataSources == nil { + c.DataSources = DSources{} + } + c.DataSources[k] = ds + } + for _, d := range contexts { + k, ds, err := parseDatasourceArg(d) + if err != nil { + return err + } + if c.Context == nil { + c.Context = DSources{} + } + c.Context[k] = ds + } + + hdrs, err := parseHeaderArgs(headers) + if err != nil { + return err + } + + for k, v := range hdrs { + if d, ok := c.Context[k]; ok { + d.Header = v + c.Context[k] = d + delete(hdrs, k) + } + if d, ok := c.DataSources[k]; ok { + d.Header = v + c.DataSources[k] = d + delete(hdrs, k) + } + } + if len(hdrs) > 0 { + c.ExtraHeaders = hdrs + } + return nil +} + +// ParsePluginFlags - sets the Plugins field from the +// key=value format flags as provided at the command-line +func (c *Config) ParsePluginFlags(plugins []string) error { + for _, plugin := range plugins { + parts := strings.SplitN(plugin, "=", 2) + if len(parts) < 2 { + return fmt.Errorf("plugin requires both name and path") + } + if c.Plugins == nil { + c.Plugins = map[string]string{} + } + c.Plugins[parts[0]] = parts[1] + } + return nil +} + +func parseDatasourceArg(value string) (key string, ds DSConfig, err error) { + parts := strings.SplitN(value, "=", 2) + if len(parts) == 1 { + f := parts[0] + key = strings.SplitN(value, ".", 2)[0] + if path.Base(f) != f { + err = fmt.Errorf("invalid datasource (%s): must provide an alias with files not in working directory", value) + return key, ds, err + } + ds.URL, err = absFileURL(f) + } else if len(parts) == 2 { + key = parts[0] + ds.URL, err = parseSourceURL(parts[1]) + } + return key, ds, err +} + +func parseHeaderArgs(headerArgs []string) (map[string]http.Header, error) { + headers := make(map[string]http.Header) + for _, v := range headerArgs { + ds, name, value, err := splitHeaderArg(v) + if err != nil { + return nil, err + } + if _, ok := headers[ds]; !ok { + headers[ds] = make(http.Header) + } + headers[ds][name] = append(headers[ds][name], strings.TrimSpace(value)) + } + return headers, nil +} + +func splitHeaderArg(arg string) (datasourceAlias, name, value string, err error) { + parts := strings.SplitN(arg, "=", 2) + if len(parts) != 2 { + err = fmt.Errorf("invalid datasource-header option '%s'", arg) + return "", "", "", err + } + datasourceAlias = parts[0] + name, value, err = splitHeader(parts[1]) + return datasourceAlias, name, value, err +} + +func splitHeader(header string) (name, value string, err error) { + parts := strings.SplitN(header, ":", 2) + if len(parts) != 2 { + err = fmt.Errorf("invalid HTTP Header format '%s'", header) + return "", "", err + } + name = http.CanonicalHeaderKey(parts[0]) + value = parts[1] + return name, value, nil +} + +// Validate the Config +func (c Config) Validate() (err error) { + err = notTogether( + []string{"in", "inputFiles", "inputDir"}, + c.Input, c.InputFiles, c.InputDir) + if err == nil { + err = notTogether( + []string{"outputFiles", "outputDir", "outputMap"}, + c.OutputFiles, c.OutputDir, c.OutputMap) + } + if err == nil { + err = notTogether( + []string{"outputDir", "outputMap", "execPipe"}, + c.OutputDir, c.OutputMap, c.ExecPipe) + } + + if err == nil { + err = mustTogether("outputDir", "inputDir", + c.OutputDir, c.InputDir) + } + + if err == nil { + err = mustTogether("outputMap", "inputDir", + c.OutputMap, c.InputDir) + } + + if err == nil { + f := len(c.InputFiles) + if f == 0 && c.Input != "" { + f = 1 + } + o := len(c.OutputFiles) + if f != o && !c.ExecPipe { + err = fmt.Errorf("must provide same number of 'outputFiles' (%d) as 'in' or 'inputFiles' (%d) options", o, f) + } + } + + if err == nil { + if c.ExecPipe && len(c.PostExec) == 0 { + err = fmt.Errorf("execPipe may only be used with a postExec command") + } + } + + if err == nil { + if c.ExecPipe && (len(c.OutputFiles) > 0 && c.OutputFiles[0] != "-") { + err = fmt.Errorf("must not set 'outputFiles' when using 'execPipe'") + } + } + + return err +} + +func notTogether(names []string, values ...interface{}) error { + found := "" + for i, value := range values { + if isZero(value) { + continue + } + if found != "" { + return fmt.Errorf("only one of these options is supported at a time: '%s', '%s'", + found, names[i]) + } + found = names[i] + } + return nil +} + +func mustTogether(left, right string, lValue, rValue interface{}) error { + if !isZero(lValue) && isZero(rValue) { + return fmt.Errorf("these options must be set together: '%s', '%s'", + left, right) + } + + return nil +} + +func isZero(value interface{}) bool { + switch v := value.(type) { + case string: + return v == "" + case []string: + return len(v) == 0 + case bool: + return !v + default: + return false + } +} + +// ApplyDefaults - +func (c *Config) ApplyDefaults() { + if c.InputDir != "" && c.OutputDir == "" && c.OutputMap == "" { + c.OutputDir = "." + } + if c.Input == "" && c.InputDir == "" && len(c.InputFiles) == 0 { + c.InputFiles = []string{"-"} + } + if c.OutputDir == "" && c.OutputMap == "" && len(c.OutputFiles) == 0 && !c.ExecPipe { + c.OutputFiles = []string{"-"} + } + if c.LDelim == "" { + c.LDelim = "{{" + } + if c.RDelim == "" { + c.RDelim = "}}" + } + + if c.ExecPipe { + c.PostExecInput = &bytes.Buffer{} + c.OutWriter = c.PostExecInput + c.OutputFiles = []string{"-"} + } else { + c.PostExecInput = os.Stdin + c.OutWriter = os.Stdout + } + + if c.PluginTimeout == 0 { + c.PluginTimeout = 5 * time.Second + } +} + +// String - +func (c *Config) String() string { + out := &strings.Builder{} + out.WriteString("---\n") + enc := yaml.NewEncoder(out) + enc.SetIndent(2) + + // dereferenced copy so we can truncate input for display + c2 := *c + if len(c2.Input) >= 11 { + c2.Input = c2.Input[0:8] + "..." + } + + err := enc.Encode(c2) + if err != nil { + return err.Error() + } + return out.String() +} + +func parseSourceURL(value string) (*url.URL, error) { + if value == "-" { + value = "stdin://" + } + value = filepath.ToSlash(value) + // handle absolute Windows paths + volName := "" + if volName = filepath.VolumeName(value); volName != "" { + // handle UNCs + if len(volName) > 2 { + value = "file:" + value + } else { + value = "file:///" + value + } + } + srcURL, err := url.Parse(value) + if err != nil { + return nil, err + } + + if volName != "" && len(srcURL.Path) >= 3 { + if srcURL.Path[0] == '/' && srcURL.Path[2] == ':' { + srcURL.Path = srcURL.Path[1:] + } + } + + if !srcURL.IsAbs() { + srcURL, err = absFileURL(value) + if err != nil { + return nil, err + } + } + return srcURL, nil +} + +func absFileURL(value string) (*url.URL, error) { + wd, err := os.Getwd() + if err != nil { + return nil, errors.Wrapf(err, "can't get working directory") + } + wd = filepath.ToSlash(wd) + baseURL := &url.URL{ + Scheme: "file", + Path: wd + "/", + } + relURL, err := url.Parse(value) + if err != nil { + return nil, fmt.Errorf("can't parse value %s as URL: %w", value, err) + } + resolved := baseURL.ResolveReference(relURL) + // deal with Windows drive letters + if !strings.HasPrefix(wd, "/") && resolved.Path[2] == ':' { + resolved.Path = resolved.Path[1:] + } + return resolved, nil +} + +// GetMode - parse an os.FileMode out of the string, and let us know if it's an override or not... +func (c *Config) GetMode() (os.FileMode, bool, error) { + modeOverride := c.OutMode != "" + m, err := strconv.ParseUint("0"+c.OutMode, 8, 32) + if err != nil { + return 0, false, err + } + mode := os.FileMode(m) + if mode == 0 && c.Input != "" { + mode = 0644 + } + return mode, modeOverride, nil +} diff --git a/internal/config/configfile_test.go b/internal/config/configfile_test.go new file mode 100644 index 000000000..b5f3104a2 --- /dev/null +++ b/internal/config/configfile_test.go @@ -0,0 +1,558 @@ +package config + +import ( + "net/http" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestParseConfigFile(t *testing.T) { + t.Parallel() + in := "in: hello world\n" + expected := &Config{ + Input: "hello world", + } + cf, err := Parse(strings.NewReader(in)) + assert.NoError(t, err) + assert.Equal(t, expected, cf) + + in = `in: hello world +outputFiles: [out.txt] +chmod: 644 + +datasources: + data: + url: file:///data.json + moredata: + url: https://example.com/more.json + header: + Authorization: ["Bearer abcd1234"] + +context: + .: + url: file:///data.json + +pluginTimeout: 2s +` + expected = &Config{ + Input: "hello world", + OutputFiles: []string{"out.txt"}, + DataSources: map[string]DSConfig{ + "data": { + URL: mustURL("file:///data.json"), + }, + "moredata": { + URL: mustURL("https://example.com/more.json"), + Header: map[string][]string{ + "Authorization": {"Bearer abcd1234"}, + }, + }, + }, + Context: map[string]DSConfig{ + ".": { + URL: mustURL("file:///data.json"), + }, + }, + OutMode: "644", + PluginTimeout: 2 * time.Second, + } + + cf, err = Parse(strings.NewReader(in)) + assert.NoError(t, err) + assert.EqualValues(t, expected, cf) +} + +func mustURL(s string) *url.URL { + u, err := url.Parse(s) + if err != nil { + panic(err) + } + // handle the case where it's a relative URL - just like in parseSourceURL. + if !u.IsAbs() { + u, err = absFileURL(s) + if err != nil { + panic(err) + } + } + return u +} + +func TestValidate(t *testing.T) { + t.Parallel() + assert.NoError(t, validateConfig("")) + + assert.Error(t, validateConfig(`in: foo +inputFiles: [bar] +`)) + assert.Error(t, validateConfig(`inputDir: foo +inputFiles: [bar] +`)) + assert.Error(t, validateConfig(`inputDir: foo +in: bar +`)) + + assert.Error(t, validateConfig(`outputDir: foo +outputFiles: [bar] +`)) + + assert.Error(t, validateConfig(`in: foo +outputFiles: [bar, baz] +`)) + + assert.Error(t, validateConfig(`inputFiles: [foo] +outputFiles: [bar, baz] +`)) + + assert.Error(t, validateConfig(`outputDir: foo +outputFiles: [bar] +`)) + + assert.Error(t, validateConfig(`outputDir: foo +`)) + + assert.Error(t, validateConfig(`outputMap: foo +`)) + + assert.Error(t, validateConfig(`outputMap: foo +outputFiles: [bar] +`)) + + assert.Error(t, validateConfig(`inputDir: foo +outputDir: bar +outputMap: bar +`)) + + assert.Error(t, validateConfig(`execPipe: true +`)) + assert.Error(t, validateConfig(`execPipe: true +postExec: "" +`)) + + assert.NoError(t, validateConfig(`execPipe: true +postExec: [echo, foo] +`)) + + assert.Error(t, validateConfig(`execPipe: true +outputFiles: [foo] +postExec: [echo] +`)) + + assert.NoError(t, validateConfig(`execPipe: true +inputFiles: ['-'] +postExec: [echo] +`)) + + assert.Error(t, validateConfig(`inputDir: foo +execPipe: true +outputDir: foo +postExec: [echo] +`)) + + assert.Error(t, validateConfig(`inputDir: foo +execPipe: true +outputMap: foo +postExec: [echo] +`)) +} + +func validateConfig(c string) error { + in := strings.NewReader(c) + cfg, err := Parse(in) + if err != nil { + return err + } + err = cfg.Validate() + return err +} + +func TestMergeFrom(t *testing.T) { + t.Parallel() + cfg := &Config{ + Input: "hello world", + DataSources: map[string]DSConfig{ + "data": { + URL: mustURL("file:///data.json"), + }, + "moredata": { + URL: mustURL("https://example.com/more.json"), + Header: http.Header{ + "Authorization": {"Bearer abcd1234"}, + }, + }, + }, + Context: map[string]DSConfig{ + "foo": { + URL: mustURL("https://example.com/foo.yaml"), + Header: http.Header{ + "Accept": {"application/yaml"}, + }, + }, + }, + OutMode: "644", + } + other := &Config{ + OutputFiles: []string{"out.txt"}, + DataSources: map[string]DSConfig{ + "data": { + Header: http.Header{ + "Accept": {"foo/bar"}, + }, + }, + }, + Context: map[string]DSConfig{ + "foo": { + Header: http.Header{ + "Accept": {"application/json"}, + }, + }, + "bar": {URL: mustURL("stdin:///")}, + }, + } + expected := &Config{ + Input: "hello world", + OutputFiles: []string{"out.txt"}, + DataSources: map[string]DSConfig{ + "data": { + URL: mustURL("file:///data.json"), + Header: http.Header{ + "Accept": {"foo/bar"}, + }, + }, + "moredata": { + URL: mustURL("https://example.com/more.json"), + Header: http.Header{ + "Authorization": {"Bearer abcd1234"}, + }, + }, + }, + Context: map[string]DSConfig{ + "foo": { + URL: mustURL("https://example.com/foo.yaml"), + Header: http.Header{ + "Accept": {"application/json"}, + }, + }, + "bar": {URL: mustURL("stdin:///")}, + }, + OutMode: "644", + } + + assert.EqualValues(t, expected, cfg.MergeFrom(other)) + + cfg = &Config{ + Input: "hello world", + } + other = &Config{ + InputFiles: []string{"in.tmpl", "in2.tmpl"}, + OutputFiles: []string{"out", "out2"}, + } + expected = &Config{ + InputFiles: []string{"in.tmpl", "in2.tmpl"}, + OutputFiles: []string{"out", "out2"}, + } + + assert.EqualValues(t, expected, cfg.MergeFrom(other)) + + cfg = &Config{ + Input: "hello world", + OutputFiles: []string{"out", "out2"}, + } + other = &Config{ + InputDir: "in/", + OutputDir: "out/", + } + expected = &Config{ + InputDir: "in/", + OutputDir: "out/", + } + + assert.EqualValues(t, expected, cfg.MergeFrom(other)) + + cfg = &Config{ + Input: "hello world", + OutputFiles: []string{"out"}, + } + other = &Config{ + Input: "hi", + ExecPipe: true, + PostExec: []string{"cat"}, + } + expected = &Config{ + Input: "hi", + ExecPipe: true, + PostExec: []string{"cat"}, + } + + assert.EqualValues(t, expected, cfg.MergeFrom(other)) + + cfg = &Config{ + Input: "hello world", + OutputFiles: []string{"-"}, + Plugins: map[string]string{ + "sleep": "echo", + }, + PluginTimeout: 500 * time.Microsecond, + } + other = &Config{ + InputFiles: []string{"-"}, + OutputFiles: []string{"-"}, + Plugins: map[string]string{ + "sleep": "sleep.sh", + }, + } + expected = &Config{ + Input: "hello world", + OutputFiles: []string{"-"}, + Plugins: map[string]string{ + "sleep": "sleep.sh", + }, + PluginTimeout: 500 * time.Microsecond, + } + + assert.EqualValues(t, expected, cfg.MergeFrom(other)) +} + +func TestParseDataSourceFlags(t *testing.T) { + t.Parallel() + cfg := &Config{} + err := cfg.ParseDataSourceFlags(nil, nil, nil) + assert.NoError(t, err) + assert.EqualValues(t, &Config{}, cfg) + + cfg = &Config{} + err = cfg.ParseDataSourceFlags([]string{"foo/bar/baz.json"}, nil, nil) + assert.Error(t, err) + + cfg = &Config{} + err = cfg.ParseDataSourceFlags([]string{"baz=foo/bar/baz.json"}, nil, nil) + assert.NoError(t, err) + expected := &Config{ + DataSources: DSources{ + "baz": {URL: mustURL("foo/bar/baz.json")}, + }, + } + assert.EqualValues(t, expected, cfg, "expected: %+v\nactual: %+v\n", expected, cfg) + + cfg = &Config{} + err = cfg.ParseDataSourceFlags( + []string{"baz=foo/bar/baz.json"}, + nil, + []string{"baz=Accept: application/json"}) + assert.NoError(t, err) + assert.EqualValues(t, &Config{ + DataSources: DSources{ + "baz": { + URL: mustURL("foo/bar/baz.json"), + Header: http.Header{ + "Accept": {"application/json"}, + }, + }, + }, + }, cfg) + + cfg = &Config{} + err = cfg.ParseDataSourceFlags( + []string{"baz=foo/bar/baz.json"}, + []string{"foo=http://example.com"}, + []string{"foo=Accept: application/json", + "bar=Authorization: Basic xxxxx"}) + assert.NoError(t, err) + assert.EqualValues(t, &Config{ + DataSources: DSources{ + "baz": {URL: mustURL("foo/bar/baz.json")}, + }, + Context: DSources{ + "foo": { + URL: mustURL("http://example.com"), + Header: http.Header{ + "Accept": {"application/json"}, + }, + }, + }, + ExtraHeaders: map[string]http.Header{ + "bar": {"Authorization": {"Basic xxxxx"}}, + }, + }, cfg) +} + +func TestParsePluginFlags(t *testing.T) { + t.Parallel() + cfg := &Config{} + err := cfg.ParsePluginFlags(nil) + assert.NoError(t, err) + + cfg = &Config{} + err = cfg.ParsePluginFlags([]string{"foo=bar"}) + assert.NoError(t, err) + assert.EqualValues(t, &Config{Plugins: map[string]string{"foo": "bar"}}, cfg) +} + +func TestConfigString(t *testing.T) { + c := &Config{} + c.ApplyDefaults() + + expected := `--- +inputFiles: ['-'] +outputFiles: ['-'] +leftDelim: '{{' +rightDelim: '}}' +pluginTimeout: 5s +` + assert.Equal(t, expected, c.String()) + + c = &Config{ + LDelim: "L", + RDelim: "R", + Input: "foo", + OutputFiles: []string{"-"}, + Templates: []string{"foo=foo.t", "bar=bar.t"}, + } + expected = `--- +in: foo +outputFiles: ['-'] +leftDelim: L +rightDelim: R +templates: +- foo=foo.t +- bar=bar.t +` + assert.Equal(t, expected, c.String()) + + c = &Config{ + LDelim: "L", + RDelim: "R", + Input: "long input that should be truncated", + OutputFiles: []string{"-"}, + Templates: []string{"foo=foo.t", "bar=bar.t"}, + } + expected = `--- +in: long inp... +outputFiles: ['-'] +leftDelim: L +rightDelim: R +templates: +- foo=foo.t +- bar=bar.t +` + assert.Equal(t, expected, c.String()) + + c = &Config{ + InputDir: "in/", + OutputDir: "out/", + } + expected = `--- +inputDir: in/ +outputDir: out/ +` + + assert.Equal(t, expected, c.String()) + + c = &Config{ + InputDir: "in/", + OutputMap: "{{ .in }}", + } + expected = `--- +inputDir: in/ +outputMap: '{{ .in }}' +` + + assert.Equal(t, expected, c.String()) + + c = &Config{ + PluginTimeout: 500 * time.Millisecond, + } + expected = `--- +pluginTimeout: 500ms +` + + assert.Equal(t, expected, c.String()) +} + +func TestApplyDefaults(t *testing.T) { + t.Parallel() + cfg := &Config{} + + cfg.ApplyDefaults() + assert.EqualValues(t, []string{"-"}, cfg.InputFiles) + assert.EqualValues(t, []string{"-"}, cfg.OutputFiles) + assert.Empty(t, cfg.OutputDir) + assert.Equal(t, "{{", cfg.LDelim) + assert.Equal(t, "}}", cfg.RDelim) + + cfg = &Config{ + InputDir: "in", + } + + cfg.ApplyDefaults() + assert.Empty(t, cfg.InputFiles) + assert.Empty(t, cfg.OutputFiles) + assert.Equal(t, ".", cfg.OutputDir) + assert.Equal(t, "{{", cfg.LDelim) + assert.Equal(t, "}}", cfg.RDelim) + + cfg = &Config{ + Input: "foo", + LDelim: "<", + RDelim: ">", + } + + cfg.ApplyDefaults() + assert.Empty(t, cfg.InputFiles) + assert.EqualValues(t, []string{"-"}, cfg.OutputFiles) + assert.Empty(t, cfg.OutputDir) + assert.Equal(t, "<", cfg.LDelim) + assert.Equal(t, ">", cfg.RDelim) + + cfg = &Config{ + Input: "foo", + ExecPipe: true, + } + + cfg.ApplyDefaults() + assert.Empty(t, cfg.InputFiles) + assert.EqualValues(t, []string{"-"}, cfg.OutputFiles) + assert.Empty(t, cfg.OutputDir) + assert.True(t, cfg.ExecPipe) + + cfg = &Config{ + InputDir: "foo", + OutputMap: "bar", + } + + cfg.ApplyDefaults() + assert.Empty(t, cfg.InputFiles) + assert.Empty(t, cfg.Input) + assert.Empty(t, cfg.OutputFiles) + assert.Empty(t, cfg.OutputDir) + assert.False(t, cfg.ExecPipe) + assert.Equal(t, "bar", cfg.OutputMap) +} + +func TestGetMode(t *testing.T) { + c := &Config{} + m, o, err := c.GetMode() + assert.NoError(t, err) + assert.Equal(t, os.FileMode(0), m) + assert.False(t, o) + + c = &Config{OutMode: "755"} + m, o, err = c.GetMode() + assert.NoError(t, err) + assert.Equal(t, os.FileMode(0755), m) + assert.True(t, o) + + c = &Config{OutMode: "0755"} + m, o, err = c.GetMode() + assert.NoError(t, err) + assert.Equal(t, os.FileMode(0755), m) + assert.True(t, o) + + c = &Config{OutMode: "foo"} + _, _, err = c.GetMode() + assert.Error(t, err) +} diff --git a/plugins.go b/plugins.go index 0699fb616..b4d561884 100644 --- a/plugins.go +++ b/plugins.go @@ -3,26 +3,26 @@ package gomplate import ( "bytes" "context" - "errors" "fmt" "os" "os/exec" "os/signal" "path/filepath" "runtime" - "strings" "text/template" "time" "github.com/hairyhenderson/gomplate/v3/conv" - "github.com/hairyhenderson/gomplate/v3/env" + "github.com/hairyhenderson/gomplate/v3/internal/config" ) -func bindPlugins(ctx context.Context, plugins []string, funcMap template.FuncMap) error { - for _, p := range plugins { - plugin, err := newPlugin(ctx, p) - if err != nil { - return err +func bindPlugins(ctx context.Context, cfg *config.Config, funcMap template.FuncMap) error { + for k, v := range cfg.Plugins { + plugin := &plugin{ + ctx: ctx, + name: k, + path: v, + timeout: cfg.PluginTimeout, } if _, ok := funcMap[plugin.name]; ok { return fmt.Errorf("function %q is already bound, and can not be overridden", plugin.name) @@ -35,23 +35,10 @@ func bindPlugins(ctx context.Context, plugins []string, funcMap template.FuncMap // plugin represents a custom function that binds to an external process to be executed type plugin struct { name, path string + timeout time.Duration ctx context.Context } -func newPlugin(ctx context.Context, value string) (*plugin, error) { - parts := strings.SplitN(value, "=", 2) - if len(parts) < 2 { - return nil, errors.New("plugin requires both name and path") - } - - p := &plugin{ - ctx: ctx, - name: parts[0], - path: parts[1], - } - return p, nil -} - // builds a command that's appropriate for running scripts // nolint: gosec func (p *plugin) buildCommand(a []string) (name string, args []string) { @@ -87,12 +74,7 @@ func (p *plugin) run(args ...interface{}) (interface{}, error) { name, a := p.buildCommand(a) - t, err := time.ParseDuration(env.Getenv("GOMPLATE_PLUGIN_TIMEOUT", "5s")) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithTimeout(p.ctx, t) + ctx, cancel := context.WithTimeout(p.ctx, p.timeout) defer cancel() c := exec.CommandContext(ctx, name, a...) c.Stdin = nil @@ -112,7 +94,7 @@ func (p *plugin) run(args ...interface{}) (interface{}, error) { } }() start := time.Now() - err = c.Run() + err := c.Run() elapsed := time.Since(start) if ctx.Err() != nil { diff --git a/plugins_test.go b/plugins_test.go index 07e232598..d1292eaa5 100644 --- a/plugins_test.go +++ b/plugins_test.go @@ -7,54 +7,48 @@ import ( "gotest.tools/v3/assert" "gotest.tools/v3/assert/cmp" -) - -func TestNewPlugin(t *testing.T) { - ctx := context.TODO() - in := "foo" - _, err := newPlugin(ctx, in) - assert.ErrorContains(t, err, "") - in = "foo=/bin/bar" - out, err := newPlugin(ctx, in) - assert.NilError(t, err) - assert.Equal(t, "foo", out.name) - assert.Equal(t, "/bin/bar", out.path) -} + "github.com/hairyhenderson/gomplate/v3/internal/config" +) func TestBindPlugins(t *testing.T) { ctx := context.TODO() fm := template.FuncMap{} - in := []string{} - err := bindPlugins(ctx, in, fm) + cfg := &config.Config{ + Plugins: map[string]string{}, + } + err := bindPlugins(ctx, cfg, fm) assert.NilError(t, err) assert.DeepEqual(t, template.FuncMap{}, fm) - in = []string{"foo=bar"} - err = bindPlugins(ctx, in, fm) + cfg.Plugins = map[string]string{"foo": "bar"} + err = bindPlugins(ctx, cfg, fm) assert.NilError(t, err) assert.Check(t, cmp.Contains(fm, "foo")) - err = bindPlugins(ctx, in, fm) + err = bindPlugins(ctx, cfg, fm) assert.ErrorContains(t, err, "already bound") } func TestBuildCommand(t *testing.T) { ctx := context.TODO() data := []struct { - plugin string - args []string - expected []string + name, path string + args []string + expected []string }{ - {"foo=foo", nil, []string{"foo"}}, - {"foo=foo", []string{"bar"}, []string{"foo", "bar"}}, - {"foo=foo.bat", nil, []string{"cmd.exe", "/c", "foo.bat"}}, - {"foo=foo.cmd", []string{"bar"}, []string{"cmd.exe", "/c", "foo.cmd", "bar"}}, - {"foo=foo.ps1", []string{"bar", "baz"}, []string{"pwsh", "-File", "foo.ps1", "bar", "baz"}}, + {"foo", "foo", nil, []string{"foo"}}, + {"foo", "foo", []string{"bar"}, []string{"foo", "bar"}}, + {"foo", "foo.bat", nil, []string{"cmd.exe", "/c", "foo.bat"}}, + {"foo", "foo.cmd", []string{"bar"}, []string{"cmd.exe", "/c", "foo.cmd", "bar"}}, + {"foo", "foo.ps1", []string{"bar", "baz"}, []string{"pwsh", "-File", "foo.ps1", "bar", "baz"}}, } for _, d := range data { - p, err := newPlugin(ctx, d.plugin) - assert.NilError(t, err) + p := &plugin{ + ctx: ctx, + name: d.name, + path: d.path, + } name, args := p.buildCommand(d.args) actual := append([]string{name}, args...) assert.DeepEqual(t, d.expected, actual) diff --git a/template.go b/template.go index 775b75cd4..138625fdb 100644 --- a/template.go +++ b/template.go @@ -9,10 +9,9 @@ import ( "path/filepath" "text/template" + "github.com/hairyhenderson/gomplate/v3/internal/config" "github.com/hairyhenderson/gomplate/v3/tmpl" - "github.com/hairyhenderson/gomplate/v3/conv" - "github.com/hairyhenderson/gomplate/v3/env" "github.com/pkg/errors" "github.com/spf13/afero" @@ -84,66 +83,65 @@ func (t *tplate) loadContents() (err error) { return err } -func (t *tplate) addTarget() (err error) { +func (t *tplate) addTarget(cfg *config.Config) (err error) { if t.name == "" && t.targetPath == "" { t.targetPath = "-" } if t.target == nil { - t.target, err = openOutFile(t.targetPath, t.mode, t.modeOverride) + t.target, err = openOutFile(cfg, t.targetPath, t.mode, t.modeOverride) } return err } // gatherTemplates - gather and prepare input template(s) and output file(s) for rendering // nolint: gocyclo -func gatherTemplates(o *Config, outFileNamer func(string) (string, error)) (templates []*tplate, err error) { - o.defaults() - mode, modeOverride, err := o.getMode() +func gatherTemplates(cfg *config.Config, outFileNamer func(string) (string, error)) (templates []*tplate, err error) { + mode, modeOverride, err := cfg.GetMode() if err != nil { return nil, err } // --exec-pipe redirects standard out to the out pipe - if o.Out != nil { - Stdout = &nopWCloser{o.Out} + if cfg.OutWriter != nil { + Stdout = &nopWCloser{cfg.OutWriter} } switch { // the arg-provided input string gets a special name - case o.Input != "": + case cfg.Input != "": templates = []*tplate{{ name: "", - contents: o.Input, + contents: cfg.Input, mode: mode, modeOverride: modeOverride, - targetPath: o.OutputFiles[0], + targetPath: cfg.OutputFiles[0], }} - case o.InputDir != "": + case cfg.InputDir != "": // input dirs presume output dirs are set too - templates, err = walkDir(o.InputDir, outFileNamer, o.ExcludeGlob, mode, modeOverride) + templates, err = walkDir(cfg.InputDir, outFileNamer, cfg.ExcludeGlob, mode, modeOverride) if err != nil { return nil, err } - case o.Input == "": - templates = make([]*tplate, len(o.InputFiles)) - for i := range o.InputFiles { - templates[i], err = fileToTemplates(o.InputFiles[i], o.OutputFiles[i], mode, modeOverride) + case cfg.Input == "": + templates = make([]*tplate, len(cfg.InputFiles)) + for i := range cfg.InputFiles { + templates[i], err = fileToTemplates(cfg.InputFiles[i], cfg.OutputFiles[i], mode, modeOverride) if err != nil { return nil, err } } } - return processTemplates(templates) + return processTemplates(cfg, templates) } -func processTemplates(templates []*tplate) ([]*tplate, error) { +func processTemplates(cfg *config.Config, templates []*tplate) ([]*tplate, error) { for _, t := range templates { if err := t.loadContents(); err != nil { return nil, err } - if err := t.addTarget(); err != nil { + if err := t.addTarget(cfg); err != nil { return nil, err } } @@ -229,8 +227,8 @@ func fileToTemplates(inFile, outFile string, mode os.FileMode, modeOverride bool return tmpl, nil } -func openOutFile(filename string, mode os.FileMode, modeOverride bool) (out io.WriteCloser, err error) { - if conv.ToBool(env.Getenv("GOMPLATE_SUPPRESS_EMPTY", "false")) { +func openOutFile(cfg *config.Config, filename string, mode os.FileMode, modeOverride bool) (out io.WriteCloser, err error) { + if cfg.SuppressEmpty { out = newEmptySkipper(func() (io.WriteCloser, error) { if filename == "-" { return Stdout, nil diff --git a/template_test.go b/template_test.go index 490917761..5f10530dc 100644 --- a/template_test.go +++ b/template_test.go @@ -7,6 +7,7 @@ import ( "os" "testing" + "github.com/hairyhenderson/gomplate/v3/internal/config" "github.com/spf13/afero" "github.com/stretchr/testify/assert" @@ -44,7 +45,8 @@ func TestOpenOutFile(t *testing.T) { fs = afero.NewMemMapFs() _ = fs.Mkdir("/tmp", 0777) - _, err := openOutFile("/tmp/foo", 0644, false) + cfg := &config.Config{} + _, err := openOutFile(cfg, "/tmp/foo", 0644, false) assert.NoError(t, err) i, err := fs.Stat("/tmp/foo") assert.NoError(t, err) @@ -53,7 +55,7 @@ func TestOpenOutFile(t *testing.T) { defer func() { Stdout = os.Stdout }() Stdout = &nopWCloser{&bytes.Buffer{}} - f, err := openOutFile("-", 0644, false) + f, err := openOutFile(cfg, "-", 0644, false) assert.NoError(t, err) assert.Equal(t, Stdout, f) } @@ -76,8 +78,9 @@ func TestAddTarget(t *testing.T) { defer func() { fs = origfs }() fs = afero.NewMemMapFs() + cfg := &config.Config{} tmpl := &tplate{name: "foo", targetPath: "/out/outfile"} - err := tmpl.addTarget() + err := tmpl.addTarget(cfg) assert.NoError(t, err) assert.NotNil(t, tmpl.target) } @@ -92,19 +95,23 @@ func TestGatherTemplates(t *testing.T) { afero.WriteFile(fs, "in/2", []byte("bar"), 0644) afero.WriteFile(fs, "in/3", []byte("baz"), 0644) - templates, err := gatherTemplates(&Config{}, nil) + cfg := &config.Config{} + cfg.ApplyDefaults() + templates, err := gatherTemplates(cfg, nil) assert.NoError(t, err) assert.Len(t, templates, 1) - templates, err = gatherTemplates(&Config{ + cfg = &config.Config{ Input: "foo", - }, nil) + } + cfg.ApplyDefaults() + templates, err = gatherTemplates(cfg, nil) assert.NoError(t, err) assert.Len(t, templates, 1) assert.Equal(t, "foo", templates[0].contents) assert.Equal(t, Stdout, templates[0].target) - templates, err = gatherTemplates(&Config{ + templates, err = gatherTemplates(&config.Config{ Input: "foo", OutputFiles: []string{"out"}, }, nil) @@ -117,7 +124,7 @@ func TestGatherTemplates(t *testing.T) { assert.Equal(t, os.FileMode(0644), info.Mode()) fs.Remove("out") - templates, err = gatherTemplates(&Config{ + templates, err = gatherTemplates(&config.Config{ InputFiles: []string{"foo"}, OutputFiles: []string{"out"}, }, nil) @@ -131,7 +138,7 @@ func TestGatherTemplates(t *testing.T) { assert.Equal(t, os.FileMode(0600), info.Mode()) fs.Remove("out") - templates, err = gatherTemplates(&Config{ + templates, err = gatherTemplates(&config.Config{ InputFiles: []string{"foo"}, OutputFiles: []string{"out"}, OutMode: "755", @@ -146,7 +153,7 @@ func TestGatherTemplates(t *testing.T) { assert.Equal(t, os.FileMode(0755), info.Mode()) fs.Remove("out") - templates, err = gatherTemplates(&Config{ + templates, err = gatherTemplates(&config.Config{ InputDir: "in", OutputDir: "out", }, simpleNamer("out")) @@ -168,6 +175,7 @@ func TestProcessTemplates(t *testing.T) { afero.WriteFile(fs, "existing", []byte(""), 0644) + cfg := &config.Config{} testdata := []struct { templates []*tplate contents []string @@ -221,7 +229,7 @@ func TestProcessTemplates(t *testing.T) { }, } for _, in := range testdata { - actual, err := processTemplates(in.templates) + actual, err := processTemplates(cfg, in.templates) assert.NoError(t, err) assert.Len(t, actual, len(in.templates)) for i, a := range actual { diff --git a/tests/integration/basic_test.go b/tests/integration/basic_test.go index 39ce824d8..2c4005e16 100644 --- a/tests/integration/basic_test.go +++ b/tests/integration/basic_test.go @@ -74,7 +74,7 @@ func (s *BasicSuite) TestErrorsWithInputOutputImbalance(c *C) { }) result.Assert(c, icmd.Expected{ ExitCode: 1, - Err: "must provide same number of --out (1) as --file (2) options", + Err: "must provide same number of 'outputFiles' (1) as 'in' or 'inputFiles' (2) options", }) } @@ -114,37 +114,37 @@ func (s *BasicSuite) TestFlagRules(c *C) { result := icmd.RunCommand(GomplateBin, "-f", "-", "-i", "HELLO WORLD") result.Assert(c, icmd.Expected{ ExitCode: 1, - Err: "only one of these flags is supported at a time: --in, --file, --input-dir", + Err: "only one of these options is supported at a time: 'in', 'inputFiles'", }) result = icmd.RunCommand(GomplateBin, "--output-dir", ".") result.Assert(c, icmd.Expected{ ExitCode: 1, - Err: "--input-dir must be set when --output-dir is set", + Err: "these options must be set together: 'outputDir', 'inputDir'", }) result = icmd.RunCommand(GomplateBin, "--input-dir", ".", "--in", "param") result.Assert(c, icmd.Expected{ ExitCode: 1, - Err: "only one of these flags is supported at a time: --in, --file, --input-dir", + Err: "only one of these options is supported at a time: 'in', 'inputDir'", }) result = icmd.RunCommand(GomplateBin, "--input-dir", ".", "--file", "input.txt") result.Assert(c, icmd.Expected{ ExitCode: 1, - Err: "only one of these flags is supported at a time: --in, --file, --input-dir", + Err: "only one of these options is supported at a time: 'inputFiles', 'inputDir'", }) result = icmd.RunCommand(GomplateBin, "--output-dir", ".", "--out", "param") result.Assert(c, icmd.Expected{ ExitCode: 1, - Err: "only one of these flags is supported at a time: --out, --output-dir, --output-map", + Err: "only one of these options is supported at a time: 'outputFiles', 'outputDir'", }) result = icmd.RunCommand(GomplateBin, "--output-map", ".", "--out", "param") result.Assert(c, icmd.Expected{ ExitCode: 1, - Err: "only one of these flags is supported at a time: --out, --output-dir, --output-map", + Err: "only one of these options is supported at a time: 'outputFiles', 'outputMap'", }) } diff --git a/tests/integration/config_test.go b/tests/integration/config_test.go new file mode 100644 index 000000000..e454dff6d --- /dev/null +++ b/tests/integration/config_test.go @@ -0,0 +1,227 @@ +//+build integration + +package integration + +import ( + "bytes" + "io/ioutil" + "os" + "runtime" + + "gopkg.in/check.v1" + + "gotest.tools/v3/assert" + "gotest.tools/v3/fs" + "gotest.tools/v3/icmd" +) + +type ConfigSuite struct { + tmpDir *fs.Dir +} + +var _ = check.Suite(&ConfigSuite{}) + +func (s *ConfigSuite) SetUpTest(c *check.C) { + s.tmpDir = fs.NewDir(c, "gomplate-inttests", + fs.WithDir("indir"), + fs.WithDir("outdir"), + fs.WithFile(".gomplate.yaml", "in: hello world\n"), + fs.WithFile("sleep.sh", "#!/bin/sh\n\nexec sleep $1\n", fs.WithMode(0755)), + ) +} + +func (s *ConfigSuite) writeFile(f, content string) { + f = s.tmpDir.Join(f) + err := ioutil.WriteFile(f, []byte(content), 0600) + if err != nil { + panic(err) + } +} + +func (s *ConfigSuite) writeConfig(content string) { + s.writeFile(".gomplate.yaml", content) +} + +func (s *ConfigSuite) TearDownTest(c *check.C) { + s.tmpDir.Remove() +} + +func (s *ConfigSuite) TestReadsFromSimpleConfigFile(c *check.C) { + result := icmd.RunCmd(icmd.Command(GomplateBin), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "hello world"}) +} + +func (s *ConfigSuite) TestReadsStdin(c *check.C) { + s.writeConfig("inputFiles: [-]") + result := icmd.RunCmd(icmd.Command(GomplateBin), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + cmd.Stdin = bytes.NewBufferString("foo bar") + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "foo bar"}) +} + +func (s *ConfigSuite) TestFlagOverridesConfig(c *check.C) { + s.writeConfig("inputFiles: [in]") + result := icmd.RunCmd(icmd.Command(GomplateBin, "-i", "hello from the cli"), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "hello from the cli"}) +} + +func (s *ConfigSuite) TestReadsFromInputFile(c *check.C) { + s.writeConfig("inputFiles: [in]") + s.writeFile("in", "blah blah") + result := icmd.RunCmd(icmd.Command(GomplateBin), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "blah blah"}) +} + +func (s *ConfigSuite) TestDatasource(c *check.C) { + s.writeConfig(`inputFiles: [in] +datasources: + data: + url: in.yaml +`) + s.writeFile("in", `{{ (ds "data").value }}`) + s.writeFile("in.yaml", `value: hello world`) + result := icmd.RunCmd(icmd.Command(GomplateBin), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "hello world"}) +} + +func (s *ConfigSuite) TestOutputDir(c *check.C) { + s.writeConfig(`inputDir: indir/ +outputDir: outdir/ +datasources: + data: + url: in.yaml +`) + s.writeFile("indir/file", `{{ (ds "data").value }}`) + s.writeFile("in.yaml", `value: hello world`) + result := icmd.RunCmd(icmd.Command(GomplateBin), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + }) + result.Assert(c, icmd.Expected{ExitCode: 0}) + b, err := ioutil.ReadFile(s.tmpDir.Join("outdir", "file")) + assert.NilError(c, err) + assert.Equal(c, "hello world", string(b)) +} + +func (s *ConfigSuite) TestExecPipeOverridesConfigFile(c *check.C) { + // make sure exec-pipe works, and outFiles is replaced + s.writeConfig(`in: hello world +outputFiles: ['-'] +`) + result := icmd.RunCmd(icmd.Command(GomplateBin, "-i", "hi", "--exec-pipe", "--", "tr", "[a-z]", "[A-Z]"), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "HI"}) +} + +func (s *ConfigSuite) TestOutFile(c *check.C) { + s.writeConfig(`in: hello world +outputFiles: [out] +`) + result := icmd.RunCmd(icmd.Command(GomplateBin), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + }) + result.Assert(c, icmd.Expected{ExitCode: 0}) + b, err := ioutil.ReadFile(s.tmpDir.Join("out")) + assert.NilError(c, err) + assert.Equal(c, "hello world", string(b)) +} + +func (s *ConfigSuite) TestAlternateConfigFile(c *check.C) { + s.writeFile("config.yaml", `in: this is from an alternate config +`) + result := icmd.RunCmd(icmd.Command(GomplateBin, "--config=config.yaml"), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "this is from an alternate config"}) +} + +func (s *ConfigSuite) TestEnvConfigFile(c *check.C) { + s.writeFile("envconfig.yaml", `in: yet another alternate config +`) + result := icmd.RunCmd(icmd.Command(GomplateBin), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + cmd.Env = []string{"GOMPLATE_CONFIG=./envconfig.yaml"} + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "yet another alternate config"}) +} + +func (s *ConfigSuite) TestConfigOverridesEnvDelim(c *check.C) { + if runtime.GOOS != "windows" { + s.writeConfig(`inputFiles: [in] +leftDelim: (╯°□°)╯︵ ┻━┻ +datasources: + data: + url: in.yaml +`) + s.writeFile("in", `(╯°□°)╯︵ ┻━┻ (ds "data").value }}`) + s.writeFile("in.yaml", `value: hello world`) + result := icmd.RunCmd(icmd.Command(GomplateBin), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + cmd.Env = []string{"GOMPLATE_LEFT_DELIM", "<<"} + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "hello world"}) + } +} + +func (s *ConfigSuite) TestFlagOverridesAllDelim(c *check.C) { + if runtime.GOOS != "windows" { + s.writeConfig(`inputFiles: [in] +leftDelim: (╯°□°)╯︵ ┻━┻ +datasources: + data: + url: in.yaml +`) + s.writeFile("in", `{{ (ds "data").value }}`) + s.writeFile("in.yaml", `value: hello world`) + result := icmd.RunCmd(icmd.Command(GomplateBin, "--left-delim={{"), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + cmd.Env = []string{"GOMPLATE_LEFT_DELIM", "<<"} + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "hello world"}) + } +} + +func (s *ConfigSuite) TestConfigOverridesEnvPluginTimeout(c *check.C) { + if runtime.GOOS != "windows" { + s.writeConfig(`in: hi there {{ sleep 2 }} +plugins: + sleep: echo + +pluginTimeout: 500ms +`) + result := icmd.RunCmd(icmd.Command(GomplateBin, + "--plugin", "sleep="+s.tmpDir.Join("sleep.sh"), + ), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + cmd.Env = []string{"GOMPLATE_PLUGIN_TIMEOUT=5s"} + }) + result.Assert(c, icmd.Expected{ExitCode: 1, Err: "plugin timed out"}) + } +} + +func (s *ConfigSuite) TestConfigOverridesEnvSuppressEmpty(c *check.C) { + s.writeConfig(`in: | + {{- print "\t \n\n\r\n\t\t \v\n" -}} + + {{ print " " -}} +out: ./missing +suppressEmpty: true +`) + result := icmd.RunCmd(icmd.Command(GomplateBin), func(cmd *icmd.Cmd) { + cmd.Dir = s.tmpDir.Path() + // should have no effect, as config overrides + cmd.Env = []string{"GOMPLATE_SUPPRESS_EMPTY=false"} + }) + result.Assert(c, icmd.Expected{ExitCode: 0}) + _, err := os.Stat(s.tmpDir.Join("missing")) + assert.Equal(c, true, os.IsNotExist(err)) +} diff --git a/tests/integration/tmpl_test.go b/tests/integration/tmpl_test.go index 4669df298..93e271486 100644 --- a/tests/integration/tmpl_test.go +++ b/tests/integration/tmpl_test.go @@ -96,7 +96,6 @@ func (s *TmplSuite) TestExec(c *C) { }) result.Assert(c, icmd.Expected{ExitCode: 0}) assert.Equal(c, "", result.Stdout()) - assert.Equal(c, "", result.Stderr()) out, err := ioutil.ReadFile(s.tmpDir.Join("out", "users", "config.json")) assert.NilError(c, err)