diff --git a/.golangci.yml b/.golangci.yml index 63290bcb..d333ee6d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -28,7 +28,7 @@ linters: - goimports - predeclared - gosec - - golint + - revive - nolintlint - unconvert - errcheck diff --git a/api/api_test.go b/api/api_test.go index 8cfff759..ad38b66a 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -26,7 +26,9 @@ import ( "github.com/ovh/utask/engine" "github.com/ovh/utask/engine/input" "github.com/ovh/utask/engine/step" + "github.com/ovh/utask/engine/step/condition" "github.com/ovh/utask/engine/step/executor" + "github.com/ovh/utask/engine/values" "github.com/ovh/utask/models/task" "github.com/ovh/utask/models/tasktemplate" "github.com/ovh/utask/pkg/auth" @@ -276,6 +278,91 @@ func TestPasswordInput(t *testing.T) { tester.Run() } +func TestResolutionResolveVar(t *testing.T) { + tester := iffy.NewTester(t, hdl) + + dbp, err := zesty.NewDBProvider(utask.DBName) + if err != nil { + t.Fatal(err) + } + + tmpl := clientErrorTemplate() + + _, err = tasktemplate.LoadFromName(dbp, tmpl.Name) + if err != nil { + if !errors.IsNotFound(err) { + t.Fatal(err) + } + if err := dbp.DB().Insert(&tmpl); err != nil { + t.Fatal(err) + } + } + + tester.AddCall("getTemplate", http.MethodGet, "/template/"+tmpl.Name, ""). + Headers(regularHeaders). + Checkers( + iffy.ExpectStatus(200), + ) + + tester.AddCall("newTask", http.MethodPost, "/task", `{"template_name":"{{.getTemplate.name}}","input":{"id":"foobarbuzz"}}`). + Headers(regularHeaders). + Checkers(iffy.ExpectStatus(201)) + + tester.AddCall("createResolution", http.MethodPost, "/resolution", `{"task_id":"{{.newTask.id}}"}`). + Headers(adminHeaders). + Checkers(iffy.ExpectStatus(201)) + + tester.AddCall("runResolution", http.MethodPost, "/resolution/{{.createResolution.id}}/run", ""). + Headers(adminHeaders). + Checkers( + iffy.ExpectStatus(204), + waitChecker(time.Second), // fugly... need to give resolution manager some time to asynchronously finish running + ) + + tester.AddCall("getResolution", http.MethodGet, "/resolution/{{.createResolution.id}}", ""). + Headers(adminHeaders). + Checkers( + iffy.ExpectStatus(200), + iffy.ExpectJSONBranch("state", "BLOCKED_BADREQUEST"), + ) + + tester.AddCall("getResolvedValuesError", http.MethodPost, "/resolution/{{.createResolution.id}}/templating", `{}`). + Headers(adminHeaders). + Checkers( + iffy.ExpectStatus(400), + ) + + tester.AddCall("getResolvedValues1", http.MethodPost, "/resolution/{{.createResolution.id}}/templating", `{"template_str":"{{ "{{" }}.input.id}}"}`). + Headers(adminHeaders). + Checkers( + iffy.ExpectStatus(200), + iffy.ExpectJSONBranch("result", "foobarbuzz"), + ) + + tester.AddCall("getResolvedValues2", http.MethodPost, "/resolution/{{.createResolution.id}}/templating", `{"template_str":"{{ "{{" }} eval \"var1\" }}"}`). + Headers(adminHeaders). + Checkers( + iffy.ExpectStatus(200), + iffy.ExpectJSONBranch("result", "hello id foobarbuzz for bar and BROKEN_TEMPLATING"), + ) + + tester.AddCall("getResolvedValues3", http.MethodPost, "/resolution/{{.createResolution.id}}/templating", `{"template_str":"{{ "{{" }} eval \"var1\" }}","step_name":"step2"}`). + Headers(adminHeaders). + Checkers( + iffy.ExpectStatus(200), + iffy.ExpectJSONBranch("result", "hello id foobarbuzz for bar and CLIENT_ERROR"), + ) + + tester.AddCall("getResolvedValues4", http.MethodPost, "/resolution/{{.createResolution.id}}/templating", `{"template_str":"{{ "{{" }} eval \"var2\" }}"}`). + Headers(adminHeaders). + Checkers( + iffy.ExpectStatus(200), + iffy.ExpectJSONBranch("result", "5"), + ) + + tester.Run() +} + func TestPagination(t *testing.T) { tester := iffy.NewTester(t, hdl) @@ -450,40 +537,6 @@ func waitChecker(dur time.Duration) iffy.Checker { } } -func templatesWithInvalidInputs() []tasktemplate.TaskTemplate { - var tt []tasktemplate.TaskTemplate - for _, inp := range []input.Input{ - { - Name: "input-with-redundant-regex", - LegalValues: []interface{}{"a", "b", "c"}, - Regex: strPtr("^d.+$"), - }, - { - Name: "input-with-bad-regex", - Regex: strPtr("^^[d.+$"), - }, - { - Name: "input-with-bad-type", - Type: "bad-type", - }, - { - Name: "input-with-bad-legal-values", - Type: "number", - LegalValues: []interface{}{"a", "b", "c"}, - }, - } { - tt = append(tt, tasktemplate.TaskTemplate{ - Name: "invalid-template", - Description: "Invalid template", - TitleFormat: "Invalid template", - Inputs: []input.Input{ - inp, - }, - }) - } - return tt -} - func templateWithPasswordInput() tasktemplate.TaskTemplate { return tasktemplate.TaskTemplate{ Name: "input-password", @@ -534,6 +587,63 @@ func dummyTemplate() tasktemplate.TaskTemplate { } } +func clientErrorTemplate() tasktemplate.TaskTemplate { + return tasktemplate.TaskTemplate{ + Name: "client-error-template", + Description: "does nothing", + TitleFormat: "this task does nothing at all", + Inputs: []input.Input{ + { + Name: "id", + }, + }, + Variables: []values.Variable{ + { + Name: "var1", + Value: "hello id {{.input.id }} for {{ .step.step1.output.foo }} and {{ .step.this.state | default \"BROKEN_TEMPLATING\" }}", + }, + { + Name: "var2", + Expression: "var a = 3+2; a;", + }, + }, + Steps: map[string]*step.Step{ + "step1": { + Action: executor.Executor{ + Type: "echo", + Configuration: json.RawMessage(`{ + "output": {"foo":"bar"} + }`), + }, + }, + "step2": { + Action: executor.Executor{ + Type: "echo", + Configuration: json.RawMessage(`{ + "output": {"foo":"bar"} + }`), + }, + Dependencies: []string{"step1"}, + Conditions: []*condition.Condition{ + { + If: []*condition.Assert{ + { + Expected: "1", + Value: "1", + Operator: "EQ", + }, + }, + Then: map[string]string{ + "this": "CLIENT_ERROR", + }, + Type: "skip", + }, + }, + }, + }, + } +} + func blockedHidden(name string, blocked, hidden bool) tasktemplate.TaskTemplate { return tasktemplate.TaskTemplate{ Name: name, @@ -587,12 +697,4 @@ func expectStringPresent(value string) iffy.Checker { } } -func marshalJSON(t *testing.T, i interface{}) string { - jsonBytes, err := json.Marshal(i) - if err != nil { - t.Fatal(err) - } - return string(jsonBytes) -} - func strPtr(s string) *string { return &s } diff --git a/api/handler/resolution.go b/api/handler/resolution.go index 72acbe17..4fdd7fad 100644 --- a/api/handler/resolution.go +++ b/api/handler/resolution.go @@ -9,6 +9,7 @@ import ( "github.com/loopfz/gadgeto/zesty" "github.com/sirupsen/logrus" + "github.com/ovh/configstore" "github.com/ovh/utask" "github.com/ovh/utask/engine" "github.com/ovh/utask/engine/step" @@ -901,3 +902,80 @@ func UpdateResolutionStepState(c *gin.Context, in *updateResolutionStepStateIn) return nil } + +type resolveTemplatingResolutionIn struct { + PublicID string `path:"id" validate:"required"` + TemplateStr string `json:"template_str" validate:"required"` + StepName string `json:"step_name"` +} + +// ResolveTemplatingResolutionOut is the output of the HTTP route +// for ResolveTemplatingResolution +type ResolveTemplatingResolutionOut struct { + Result string `json:"result"` + Error *string `json:"error"` +} + +// ResolveTemplatingResolution will use µtask templating engine for a given resolution +// to validate a given template. Action is restricted to admin only, as it could be used +// to exfiltrate configuration. +func ResolveTemplatingResolution(c *gin.Context, in *resolveTemplatingResolutionIn) (*ResolveTemplatingResolutionOut, error) { + metadata.AddActionMetadata(c, metadata.ResolutionID, in.PublicID) + + dbp, err := zesty.NewDBProvider(utask.DBName) + if err != nil { + return nil, err + } + + r, err := resolution.LoadFromPublicID(dbp, in.PublicID) + if err != nil { + return nil, err + } + + t, err := task.LoadFromID(dbp, r.TaskID) + if err != nil { + return nil, err + } + + metadata.AddActionMetadata(c, metadata.TaskID, t.PublicID) + + tt, err := tasktemplate.LoadFromID(dbp, t.TemplateID) + if err != nil { + return nil, err + } + + metadata.AddActionMetadata(c, metadata.TemplateName, tt.Name) + + admin := auth.IsAdmin(c) == nil + + if !admin { + return nil, errors.Forbiddenf("You are not allowed to resolve resolution variables") + } + + metadata.SetSUDO(c) + + // provide the resolution with values + t.ExportTaskInfos(r.Values) + r.Values.SetInput(t.Input) + r.Values.SetResolverInput(r.ResolverInput) + r.Values.SetVariables(tt.Variables) + + config, err := utask.GetTemplatingConfig(configstore.DefaultStore) + if err != nil { + return nil, err + } + + r.Values.SetConfig(config) + + output, err := r.Values.Apply(in.TemplateStr, nil, in.StepName) + if err != nil { + errStr := err.Error() + return &ResolveTemplatingResolutionOut{ + Error: &errStr, + }, nil + } + + return &ResolveTemplatingResolutionOut{ + Result: string(output), + }, nil +} diff --git a/api/server.go b/api/server.go index 414eda3b..e7f8c6af 100644 --- a/api/server.go +++ b/api/server.go @@ -401,6 +401,14 @@ func (s *Server) build(ctx context.Context) { }, maintenanceMode, tonic.Handler(handler.UpdateResolutionStepState, 204)) + resolutionRoutes.POST("/resolution/:id/templating", + []fizz.OperationOption{ + fizz.ID("ResolveTemplatingResolution"), + fizz.Summary("Resolve templating of a resolution"), + fizz.Description("Resolve the templating of a string, within a task resolution. Admin users only."), + }, + maintenanceMode, + tonic.Handler(handler.ResolveTemplatingResolution, 200)) // resolutionRoutes.POST("/resolution/:id/rollback", // []fizz.OperationOption{ diff --git a/engine/engine.go b/engine/engine.go index 0dc3b913..76d890e0 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -3,14 +3,12 @@ package engine import ( "bytes" "context" - "encoding/json" "fmt" "strings" "sync" "time" "github.com/cenkalti/backoff" - "github.com/ghodss/yaml" expbk "github.com/jpillora/backoff" "github.com/juju/errors" "github.com/loopfz/gadgeto/zesty" @@ -65,38 +63,13 @@ func Init(ctx context.Context, wg *sync.WaitGroup, store *configstore.Store) err if err != nil { return err } - // get all configuration items - itemList, err := store.GetItemList() - if err != nil { - return err - } - // Squash to ensure that secrets with lower priority - // are dismissed. - itemList = configstore.Filter().Squash().Apply(itemList) - // drop those that shouldnt be available for task execution - // (don't let DB credentials leak, for instance...) - config, err := filteredConfig(itemList, cfg.ConcealedSecrets...) - if err != nil { + var engineCfg map[string]interface{} + if engineCfg, err = utask.GetTemplatingConfig(store); err != nil { return err } - // attempt to deserialize json formatted config items - // -> make it easier to access internal nodes/values when templating - eng.config = make(map[string]interface{}) - for k, v := range config { - var i interface{} - if v != nil { - err := yaml.Unmarshal([]byte(*v), &i, func(dec *json.Decoder) *json.Decoder { - dec.UseNumber() - return dec - }) - if err != nil { - eng.config[k] = v - } else { - eng.config[k] = i - } - } - } + + eng.config = engineCfg // channels for handling graceful shutdown shutdownCtx = ctx @@ -151,27 +124,6 @@ func Init(ctx context.Context, wg *sync.WaitGroup, store *configstore.Store) err return nil } -// filteredConfig takes a configstore item list, drops some items by key -// then reduces the result into a map of key->values -func filteredConfig(list *configstore.ItemList, dropAlias ...string) (map[string]*string, error) { - cfg := make(map[string]*string) - for _, i := range list.Items { - if !utils.ListContainsString(dropAlias, i.Key()) { - // assume only one value per alias - if _, ok := cfg[i.Key()]; !ok { - v, err := i.Value() - if err != nil { - return nil, err - } - if len(v) > 0 { - cfg[i.Key()] = &v - } - } - } - } - return cfg, nil -} - // GetEngine returns the singleton instance of Engine func GetEngine() Engine { return eng diff --git a/utask.go b/utask.go index c786d747..97346080 100644 --- a/utask.go +++ b/utask.go @@ -7,6 +7,7 @@ import ( "fmt" "time" + "github.com/ghodss/yaml" "golang.org/x/sync/semaphore" "github.com/ovh/configstore" @@ -386,3 +387,76 @@ func Config(store *configstore.Store) (*Cfg, error) { return global, nil } + +// Init launches the task orchestration engine, providing it with a global context +// and with a store from which to inherit configuration items needed for task execution +func GetTemplatingConfig(store *configstore.Store) (map[string]interface{}, error) { + cfg, err := Config(store) + if err != nil { + return nil, err + } + // get all configuration items + itemList, err := store.GetItemList() + if err != nil { + return nil, err + } + // Squash to ensure that secrets with lower priority + // are dismissed. + itemList = configstore.Filter().Squash().Apply(itemList) + + // drop those that shouldnt be available for task execution + // (don't let DB credentials leak, for instance...) + config, err := filteredConfig(itemList, cfg.ConcealedSecrets...) + if err != nil { + return nil, err + } + // attempt to deserialize json formatted config items + // -> make it easier to access internal nodes/values when templating + c := make(map[string]interface{}) + for k, v := range config { + var i interface{} + if v != nil { + err := yaml.Unmarshal([]byte(*v), &i, func(dec *json.Decoder) *json.Decoder { + dec.UseNumber() + return dec + }) + if err != nil { + c[k] = v + } else { + c[k] = i + } + } + } + return c, nil +} + +// filteredConfig takes a configstore item list, drops some items by key +// then reduces the result into a map of key->values +func filteredConfig(list *configstore.ItemList, dropAlias ...string) (map[string]*string, error) { + cfg := make(map[string]*string) + for _, i := range list.Items { + if !listContainsString(dropAlias, i.Key()) { + // assume only one value per alias + if _, ok := cfg[i.Key()]; !ok { + v, err := i.Value() + if err != nil { + return nil, err + } + if len(v) > 0 { + cfg[i.Key()] = &v + } + } + } + } + return cfg, nil +} + +// listContainsString asserts that a string slice contains a given string +func listContainsString(list []string, item string) bool { + for _, i := range list { + if i == item { + return true + } + } + return false +}