internal/config: use JSON Schema (#678)

This change switches the `runme.yaml` schema definition from protobuf to
JSON Schema. This results in fewer conversions and more natural
representation. It also adds support for parsing and merging multiple
configuration files. Default config was moved to [a
file](internal/config/runme.default.yaml).

Go's struct representing the config is generated from the schema. When
you change `internal/config/config.schema.json`, remember to call `make
schema/generate`.

---------

Co-authored-by: Sebastian Tiedtke <[email protected]>

.gitignore

@@ -52,3 +52,6 @@ cover.out
 # Prevent git from interpreting internal/project/testdata/git-project as a submodule.
 internal/project/testdata/git-project/**/.git
 internal/project/testdata/git-project/**/.gitignore
+
+# Ignore user-specific runme.yaml files.
+experimental/runme.*.yaml

Makefile

@@ -117,6 +117,7 @@ install/dev:
 	go install mvdan.cc/[email protected]
 	go install github.com/icholy/[email protected]
 	go install github.com/stateful/go-proto-gql/protoc-gen-gql@latest
+	go install github.com/atombender/[email protected]
 
 .PHONY: install/goreleaser
 install/goreleaser:
@@ -148,6 +149,14 @@ proto/dev/reset:
 proto/publish:
 	@cd ./pkg/api/proto && buf push
 
+.PHONY: schema/generate
+schema/generate:
+	go-jsonschema -t \
+		-p config \
+		--tags "json,yaml" \
+		-o internal/config/config_schema.go \
+		internal/config/config.schema.json
+
 .PHONY: release
 release: install/goreleaser
 	@goreleaser check

cmd/gqltool/main.go

@@ -19,10 +19,6 @@ func init() {
 }
 
 func main() {
-	// httpClient := client.NewHTTPClient(nil, client.WithTokenGetter(func() (string, error) {
-	// 	a := auth.New(oauth2.Config{}, *apiURL, &auth.DiskStorage{Location: *tokenDir})
-	// 	return a.GetToken(context.Background())
-	// }))
 	httpClient := client.NewHTTPClient(nil)
 	client, err := graphql.New(*apiURL+"/graphql", httpClient)
 	if err != nil {

docker/runme-test-env.Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -y \
     curl \
     make \
     python3 \
+    ruby-full \
     unzip
 
 # Install node.js

go.mod

@@ -5,11 +5,9 @@ go 1.22.0
 // replace github.com/stateful/godotenv => ../godotenv
 
 require (
-	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/Microsoft/go-winio v0.6.2
 	github.com/atotto/clipboard v0.1.4
-	github.com/bufbuild/protovalidate-go v0.6.3
 	github.com/charmbracelet/bubbletea v0.26.6
 	github.com/charmbracelet/lipgloss v0.12.1
 	github.com/cli/cli/v2 v2.53.0
@@ -53,7 +51,6 @@ require (
 
 require (
 	dario.cat/mergo v1.0.0 // indirect
-	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/bufbuild/protocompile v0.14.0 // indirect
 	github.com/charmbracelet/x/ansi v0.1.4 // indirect
@@ -74,12 +71,10 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/google/cel-go v0.20.1 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
 	go.opentelemetry.io/otel v1.28.0 // indirect

go.sum

@@ -1,5 +1,3 @@
-buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2 h1:SZRVx928rbYZ6hEKUIN+vtGDkl7uotABRWGY4OAg5gM=
-buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2/go.mod h1:ylS4c28ACSI59oJrOdW4pHS4n0Hw4TgSPHn8rpHl4Yw=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
@@ -19,8 +17,6 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
-github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
-github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
@@ -31,8 +27,6 @@ github.com/briandowns/spinner v1.23.1 h1:t5fDPmScwUjozhDj4FA46p5acZWIPXYE30qW2Pt
 github.com/briandowns/spinner v1.23.1/go.mod h1:LaZeM4wm2Ywy6vO571mvhQNRcWfRUnXOs0RcKV0wYKM=
 github.com/bufbuild/protocompile v0.14.0 h1:z3DW4IvXE5G/uTOnSQn+qwQQxvhckkTWLS/0No/o7KU=
 github.com/bufbuild/protocompile v0.14.0/go.mod h1:N6J1NYzkspJo3ZwyL4Xjvli86XOj1xq4qAasUFxGups=
-github.com/bufbuild/protovalidate-go v0.6.3 h1:wxQyzW035zM16Binbaz/nWAzS12dRIXhZdSUWRY7Fv0=
-github.com/bufbuild/protovalidate-go v0.6.3/go.mod h1:J4PtwP9Z2YAGgB0+o+tTWEDtLtXvz/gfhFZD8pbzM/U=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
@@ -143,8 +137,6 @@ github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
-github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84=
-github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -263,8 +255,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stateful/godotenv v0.0.0-20240309032207-c7bc0b812915 h1:rBwOH8hK4mnonIOv9qV76i+nhmJIMaxqUeuzg9e7pF8=
 github.com/stateful/godotenv v0.0.0-20240309032207-c7bc0b812915/go.mod h1:A7pPuRB981nGoMyu09TOEDPHzg/eVlO3rgy1pk91xYY=
-github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
-github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -274,18 +264,17 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0vy5p8=
 github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
-github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
-github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
 github.com/xo/dburl v0.23.2 h1:Fl88cvayrgE56JA/sqhNMLljCW/b7RmG1mMkKMZUFgA=
 github.com/xo/dburl v0.23.2/go.mod h1:uazlaAQxj4gkshhfuuYyvwCBouOmNnG2aDxTCFZpmL4=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=

internal/cmd/beta/beta_cmd.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	"go.uber.org/zap"
 
 	"github.com/stateful/runme/v3/internal/cmd/beta/server"
 	"github.com/stateful/runme/v3/internal/config"
@@ -36,25 +37,30 @@ All commands use the runme.yaml configuration file.`,
 				cmd.SetErr(io.Discard)
 			}
 
-			err := autoconfig.InvokeForCommand(func(cfg *config.Config) error {
+			err := autoconfig.InvokeForCommand(func(cfg *config.Config, log *zap.Logger) error {
 				// Override the filename if provided.
 				if cFlags.filename != "" {
-					cfg.ProjectFilename = cFlags.filename
+					cfg.Project.Filename = cFlags.filename
 				}
 
 				// Add a filter to run only tasks from the specified tags.
 				if len(cFlags.tags) > 0 {
-					cfg.ProjectFilters = append(cfg.ProjectFilters, &config.Filter{
-						Type:      config.FilterTypeBlock,
-						Condition: `len(intersection(tags, extra.tags)) > 0`,
-						Extra:     map[string]interface{}{"tags": cFlags.tags},
-					})
+					cfg.Project.Filters = append(
+						cfg.Project.Filters,
+						config.ConfigProjectFiltersElem{
+							Type:      config.FilterTypeBlock,
+							Condition: `len(intersection(tags, extra.tags)) > 0`,
+							Extra:     map[string]interface{}{"tags": cFlags.tags},
+						},
+					)
 				}
 
+				log.Info("final configuration", zap.Any("config", cfg))
+
 				return nil
 			})
-			// print the error to stderr but don't return it because error modes
-			// are neither fully baked yet nor ready for users to consume
+			// Print the error to stderr but don't return it because error modes
+			// are neither fully baked yet nor ready for users to consume.
 			if err != nil {
 				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%s\n", err)
 			}

internal/cmd/beta/server/grpcurl_utils.go

@@ -27,14 +27,14 @@ func getDescriptorSource(ctx context.Context, cfg *config.Config) (grpcurl.Descr
 }
 
 func dialServer(ctx context.Context, cfg *config.Config) (*grpc.ClientConn, error) {
-	tlsConf, err := runmetls.LoadClientConfig(cfg.ServerTLSCertFile, cfg.ServerTLSKeyFile)
+	tlsConf, err := runmetls.LoadClientConfig(*cfg.Server.Tls.CertFile, *cfg.Server.Tls.KeyFile)
 	if err != nil {
 		return nil, err
 	}
 
 	creds := credentials.NewTLS(tlsConf)
 
-	network, addr := "tcp", cfg.ServerAddress
+	network, addr := "tcp", cfg.Server.Address
 	if strings.HasPrefix(addr, "unix://") {
 		network, addr = "unix", strings.TrimPrefix(addr, "unix://")
 	}

internal/cmd/beta/server/server_cmd.go

@@ -19,7 +19,7 @@ func Cmd() *cobra.Command {
 				) error {
 					// For the server commands, we want to always log to stdout.
 					// TODO(adamb): there might be a need to separate client and server logs.
-					cfg.LogPath = ""
+					cfg.Log.Path = ""
 					return nil
 				},
 			)

internal/cmd/beta/server/server_start_cmd.go

@@ -28,10 +28,10 @@ func serverStartCmd() *cobra.Command {
 					defer logger.Sync()
 
 					serverCfg := &server.Config{
-						Address:    cfg.ServerAddress,
-						CertFile:   cfg.ServerTLSCertFile,
-						KeyFile:    cfg.ServerTLSKeyFile,
-						TLSEnabled: cfg.ServerTLSEnabled,
+						Address:    cfg.Server.Address,
+						CertFile:   *cfg.Server.Tls.CertFile, // guaranteed by autoconfig
+						KeyFile:    *cfg.Server.Tls.KeyFile,  // guaranteed by autoconfig
+						TLSEnabled: cfg.Server.Tls.Enabled,
 					}
 
 					_ = telemetry.ReportUnlessNoTracking(logger)
@@ -44,12 +44,12 @@ func serverStartCmd() *cobra.Command {
 					}
 
 					// When using a unix socket, we want to create a file with server's PID.
-					if path := pidFileNameFromAddr(cfg.ServerAddress); path != "" {
+					if path := pidFileNameFromAddr(cfg.Server.Address); path != "" {
 						logger.Debug("creating PID file", zap.String("path", path))
 						if err := createFileWithPID(path); err != nil {
 							return errors.WithStack(err)
 						}
-						defer os.Remove(cfg.ServerAddress)
+						defer os.Remove(cfg.Server.Address)
 					}
 
 					logger.Debug("starting the server")

internal/cmd/beta/server/server_stop_cmd.go

@@ -27,7 +27,7 @@ func serverStopCmd() *cobra.Command {
 
 					logger.Debug("stopping the server by looking for runme.pid")
 
-					path := pidFileNameFromAddr(cfg.ServerAddress)
+					path := pidFileNameFromAddr(cfg.Server.Address)
 					if path == "" {
 						return errors.New("server address is not a unix socket")
 					}