From 9e98d694a6230b904f931813b7d53622e9f128c9 Mon Sep 17 00:00:00 2001 From: Gerard Nguyen Date: Sat, 29 Jul 2023 04:53:32 +1000 Subject: [PATCH] feature: Add new field render_templates on restart block (#18054) This feature is necessary when user want to explicitly re-render all templates on task restart. E.g. to fetch all new secrets from Vault, even if the lease on the existing secrets has not been expired. --- .changelog/18054.txt | 3 + api/jobs_test.go | 106 ++++++++++-------- api/tasks.go | 30 +++-- .../taskrunner/task_runner_hooks.go | 17 +-- .../allocrunner/taskrunner/template_hook.go | 10 +- command/agent/job_endpoint.go | 18 +-- command/agent/job_endpoint_test.go | 63 ++++++----- command/agent/testingutils_test.go | 9 +- command/helpers_test.go | 14 ++- jobspec/parse_group.go | 1 + jobspec/parse_test.go | 9 +- jobspec/test-fixtures/basic.hcl | 9 +- jobspec2/parse_test.go | 18 +++ .../restart-render-templates.hcl | 17 +++ nomad/mock/job.go | 9 +- nomad/structs/diff_test.go | 51 +++++++-- nomad/structs/structs.go | 21 ++-- nomad/structs/structs_test.go | 7 +- scheduler/util.go | 11 ++ scheduler/util_test.go | 16 +++ .../docs/job-specification/restart.mdx | 41 ++++--- 21 files changed, 314 insertions(+), 166 deletions(-) create mode 100644 .changelog/18054.txt create mode 100644 jobspec2/test-fixtures/restart-render-templates.hcl diff --git a/.changelog/18054.txt b/.changelog/18054.txt new file mode 100644 index 000000000000..4a30ef96c40d --- /dev/null +++ b/.changelog/18054.txt @@ -0,0 +1,3 @@ +```release-note:improvement +jobspec: Add new parameter `render_templates` for `restart` block to allow explicit re-render of templates on task restart. The default value is `false` and is fully backward compatible +``` \ No newline at end of file diff --git a/api/jobs_test.go b/api/jobs_test.go index f17dee385712..83c20c971cac 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -320,10 +320,11 @@ func TestJobs_Canonicalize(t *testing.T) { SizeMB: pointerOf(300), }, RestartPolicy: &RestartPolicy{ - Delay: pointerOf(15 * time.Second), - Attempts: pointerOf(2), - Interval: pointerOf(30 * time.Minute), - Mode: pointerOf("fail"), + Delay: pointerOf(15 * time.Second), + Attempts: pointerOf(2), + Interval: pointerOf(30 * time.Minute), + Mode: pointerOf("fail"), + RenderTemplates: pointerOf(false), }, ReschedulePolicy: &ReschedulePolicy{ Attempts: pointerOf(0), @@ -405,10 +406,11 @@ func TestJobs_Canonicalize(t *testing.T) { SizeMB: pointerOf(300), }, RestartPolicy: &RestartPolicy{ - Delay: pointerOf(15 * time.Second), - Attempts: pointerOf(3), - Interval: pointerOf(24 * time.Hour), - Mode: pointerOf("fail"), + Delay: pointerOf(15 * time.Second), + Attempts: pointerOf(3), + Interval: pointerOf(24 * time.Hour), + Mode: pointerOf("fail"), + RenderTemplates: pointerOf(false), }, ReschedulePolicy: &ReschedulePolicy{ Attempts: pointerOf(1), @@ -495,10 +497,11 @@ func TestJobs_Canonicalize(t *testing.T) { SizeMB: pointerOf(300), }, RestartPolicy: &RestartPolicy{ - Delay: pointerOf(15 * time.Second), - Attempts: pointerOf(2), - Interval: pointerOf(30 * time.Minute), - Mode: pointerOf("fail"), + Delay: pointerOf(15 * time.Second), + Attempts: pointerOf(2), + Interval: pointerOf(30 * time.Minute), + Mode: pointerOf("fail"), + RenderTemplates: pointerOf(false), }, ReschedulePolicy: &ReschedulePolicy{ Attempts: pointerOf(0), @@ -663,10 +666,11 @@ func TestJobs_Canonicalize(t *testing.T) { Name: pointerOf("cache"), Count: pointerOf(1), RestartPolicy: &RestartPolicy{ - Interval: pointerOf(5 * time.Minute), - Attempts: pointerOf(10), - Delay: pointerOf(25 * time.Second), - Mode: pointerOf("delay"), + Interval: pointerOf(5 * time.Minute), + Attempts: pointerOf(10), + Delay: pointerOf(25 * time.Second), + Mode: pointerOf("delay"), + RenderTemplates: pointerOf(false), }, ReschedulePolicy: &ReschedulePolicy{ Attempts: pointerOf(0), @@ -707,10 +711,11 @@ func TestJobs_Canonicalize(t *testing.T) { }}, }, RestartPolicy: &RestartPolicy{ - Interval: pointerOf(5 * time.Minute), - Attempts: pointerOf(20), - Delay: pointerOf(25 * time.Second), - Mode: pointerOf("delay"), + Interval: pointerOf(5 * time.Minute), + Attempts: pointerOf(20), + Delay: pointerOf(25 * time.Second), + Mode: pointerOf("delay"), + RenderTemplates: pointerOf(false), }, Resources: &Resources{ CPU: pointerOf(500), @@ -835,7 +840,6 @@ func TestJobs_Canonicalize(t *testing.T) { }, }, }, - { name: "update_merge", input: &Job{ @@ -928,10 +932,11 @@ func TestJobs_Canonicalize(t *testing.T) { SizeMB: pointerOf(300), }, RestartPolicy: &RestartPolicy{ - Delay: pointerOf(15 * time.Second), - Attempts: pointerOf(2), - Interval: pointerOf(30 * time.Minute), - Mode: pointerOf("fail"), + Delay: pointerOf(15 * time.Second), + Attempts: pointerOf(2), + Interval: pointerOf(30 * time.Minute), + Mode: pointerOf("fail"), + RenderTemplates: pointerOf(false), }, ReschedulePolicy: &ReschedulePolicy{ Attempts: pointerOf(0), @@ -975,10 +980,11 @@ func TestJobs_Canonicalize(t *testing.T) { SizeMB: pointerOf(300), }, RestartPolicy: &RestartPolicy{ - Delay: pointerOf(15 * time.Second), - Attempts: pointerOf(2), - Interval: pointerOf(30 * time.Minute), - Mode: pointerOf("fail"), + Delay: pointerOf(15 * time.Second), + Attempts: pointerOf(2), + Interval: pointerOf(30 * time.Minute), + Mode: pointerOf("fail"), + RenderTemplates: pointerOf(false), }, ReschedulePolicy: &ReschedulePolicy{ Attempts: pointerOf(0), @@ -1016,7 +1022,6 @@ func TestJobs_Canonicalize(t *testing.T) { }, }, }, - { name: "restart_merge", input: &Job{ @@ -1036,8 +1041,9 @@ func TestJobs_Canonicalize(t *testing.T) { { Name: "task1", RestartPolicy: &RestartPolicy{ - Attempts: pointerOf(5), - Delay: pointerOf(1 * time.Second), + Attempts: pointerOf(5), + Delay: pointerOf(1 * time.Second), + RenderTemplates: pointerOf(true), }, }, }, @@ -1105,10 +1111,11 @@ func TestJobs_Canonicalize(t *testing.T) { SizeMB: pointerOf(300), }, RestartPolicy: &RestartPolicy{ - Delay: pointerOf(15 * time.Second), - Attempts: pointerOf(2), - Interval: pointerOf(30 * time.Minute), - Mode: pointerOf("fail"), + Delay: pointerOf(15 * time.Second), + Attempts: pointerOf(2), + Interval: pointerOf(30 * time.Minute), + Mode: pointerOf("fail"), + RenderTemplates: pointerOf(false), }, ReschedulePolicy: &ReschedulePolicy{ Attempts: pointerOf(0), @@ -1140,10 +1147,11 @@ func TestJobs_Canonicalize(t *testing.T) { Resources: DefaultResources(), KillTimeout: pointerOf(5 * time.Second), RestartPolicy: &RestartPolicy{ - Attempts: pointerOf(5), - Delay: pointerOf(1 * time.Second), - Interval: pointerOf(30 * time.Minute), - Mode: pointerOf("fail"), + Attempts: pointerOf(5), + Delay: pointerOf(1 * time.Second), + Interval: pointerOf(30 * time.Minute), + Mode: pointerOf("fail"), + RenderTemplates: pointerOf(true), }, }, }, @@ -1157,10 +1165,11 @@ func TestJobs_Canonicalize(t *testing.T) { SizeMB: pointerOf(300), }, RestartPolicy: &RestartPolicy{ - Delay: pointerOf(20 * time.Second), - Attempts: pointerOf(2), - Interval: pointerOf(30 * time.Minute), - Mode: pointerOf("fail"), + Delay: pointerOf(20 * time.Second), + Attempts: pointerOf(2), + Interval: pointerOf(30 * time.Minute), + Mode: pointerOf("fail"), + RenderTemplates: pointerOf(false), }, ReschedulePolicy: &ReschedulePolicy{ Attempts: pointerOf(0), @@ -1192,10 +1201,11 @@ func TestJobs_Canonicalize(t *testing.T) { Resources: DefaultResources(), KillTimeout: pointerOf(5 * time.Second), RestartPolicy: &RestartPolicy{ - Delay: pointerOf(20 * time.Second), - Attempts: pointerOf(2), - Interval: pointerOf(30 * time.Minute), - Mode: pointerOf("fail"), + Delay: pointerOf(20 * time.Second), + Attempts: pointerOf(2), + Interval: pointerOf(30 * time.Minute), + Mode: pointerOf("fail"), + RenderTemplates: pointerOf(false), }, }, }, diff --git a/api/tasks.go b/api/tasks.go index 188fa8649bb5..16dd89bfb45e 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -88,10 +88,11 @@ type AllocCheckStatuses map[string]AllocCheckStatus // RestartPolicy defines how the Nomad client restarts // tasks in a taskgroup when they fail type RestartPolicy struct { - Interval *time.Duration `hcl:"interval,optional"` - Attempts *int `hcl:"attempts,optional"` - Delay *time.Duration `hcl:"delay,optional"` - Mode *string `hcl:"mode,optional"` + Interval *time.Duration `hcl:"interval,optional"` + Attempts *int `hcl:"attempts,optional"` + Delay *time.Duration `hcl:"delay,optional"` + Mode *string `hcl:"mode,optional"` + RenderTemplates *bool `mapstructure:"render_templates" hcl:"render_templates,optional"` } func (r *RestartPolicy) Merge(rp *RestartPolicy) { @@ -107,6 +108,9 @@ func (r *RestartPolicy) Merge(rp *RestartPolicy) { if rp.Mode != nil { r.Mode = rp.Mode } + if rp.RenderTemplates != nil { + r.RenderTemplates = rp.RenderTemplates + } } // Reschedule configures how Tasks are rescheduled when they crash or fail. @@ -580,10 +584,11 @@ func (g *TaskGroup) Canonicalize(job *Job) { // in nomad/structs/structs.go func defaultServiceJobRestartPolicy() *RestartPolicy { return &RestartPolicy{ - Delay: pointerOf(15 * time.Second), - Attempts: pointerOf(2), - Interval: pointerOf(30 * time.Minute), - Mode: pointerOf(RestartPolicyModeFail), + Delay: pointerOf(15 * time.Second), + Attempts: pointerOf(2), + Interval: pointerOf(30 * time.Minute), + Mode: pointerOf(RestartPolicyModeFail), + RenderTemplates: pointerOf(false), } } @@ -591,10 +596,11 @@ func defaultServiceJobRestartPolicy() *RestartPolicy { // in nomad/structs/structs.go func defaultBatchJobRestartPolicy() *RestartPolicy { return &RestartPolicy{ - Delay: pointerOf(15 * time.Second), - Attempts: pointerOf(3), - Interval: pointerOf(24 * time.Hour), - Mode: pointerOf(RestartPolicyModeFail), + Delay: pointerOf(15 * time.Second), + Attempts: pointerOf(3), + Interval: pointerOf(24 * time.Hour), + Mode: pointerOf(RestartPolicyModeFail), + RenderTemplates: pointerOf(false), } } diff --git a/client/allocrunner/taskrunner/task_runner_hooks.go b/client/allocrunner/taskrunner/task_runner_hooks.go index 698915878bc3..22fd09066775 100644 --- a/client/allocrunner/taskrunner/task_runner_hooks.go +++ b/client/allocrunner/taskrunner/task_runner_hooks.go @@ -111,14 +111,15 @@ func (tr *TaskRunner) initHooks() { // If there are templates is enabled, add the hook if len(task.Templates) != 0 { tr.runnerHooks = append(tr.runnerHooks, newTemplateHook(&templateHookConfig{ - logger: hookLogger, - lifecycle: tr, - events: tr, - templates: task.Templates, - clientConfig: tr.clientConfig, - envBuilder: tr.envBuilder, - consulNamespace: consulNamespace, - nomadNamespace: tr.alloc.Job.Namespace, + logger: hookLogger, + lifecycle: tr, + events: tr, + templates: task.Templates, + clientConfig: tr.clientConfig, + envBuilder: tr.envBuilder, + consulNamespace: consulNamespace, + nomadNamespace: tr.alloc.Job.Namespace, + renderOnTaskRestart: task.RestartPolicy.RenderTemplates, })) } diff --git a/client/allocrunner/taskrunner/template_hook.go b/client/allocrunner/taskrunner/template_hook.go index 3f3cc8e7a59a..a2292f7b77e2 100644 --- a/client/allocrunner/taskrunner/template_hook.go +++ b/client/allocrunner/taskrunner/template_hook.go @@ -45,6 +45,9 @@ type templateHookConfig struct { // nomadNamespace is the job's Nomad namespace nomadNamespace string + + // renderOnTaskRestart is flag to explicitly render templates on task restart + renderOnTaskRestart bool } type templateHook struct { @@ -97,7 +100,12 @@ func (h *templateHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar // If we have already run prerun before exit early. if h.templateManager != nil { - return nil + if !h.config.renderOnTaskRestart { + return nil + } + h.logger.Info("re-rendering templates on task restart") + h.templateManager.Stop() + h.templateManager = nil } // Store the current Vault token and the task directory diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 68878377692e..12a4042158d0 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1058,10 +1058,11 @@ func ApiTgToStructsTG(job *structs.Job, taskGroup *api.TaskGroup, tg *structs.Ta tg.Consul = apiConsulToStructs(taskGroup.Consul) tg.RestartPolicy = &structs.RestartPolicy{ - Attempts: *taskGroup.RestartPolicy.Attempts, - Interval: *taskGroup.RestartPolicy.Interval, - Delay: *taskGroup.RestartPolicy.Delay, - Mode: *taskGroup.RestartPolicy.Mode, + Attempts: *taskGroup.RestartPolicy.Attempts, + Interval: *taskGroup.RestartPolicy.Interval, + Delay: *taskGroup.RestartPolicy.Delay, + Mode: *taskGroup.RestartPolicy.Mode, + RenderTemplates: *taskGroup.RestartPolicy.RenderTemplates, } if taskGroup.ShutdownDelay != nil { @@ -1209,10 +1210,11 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup, if apiTask.RestartPolicy != nil { structsTask.RestartPolicy = &structs.RestartPolicy{ - Attempts: *apiTask.RestartPolicy.Attempts, - Interval: *apiTask.RestartPolicy.Interval, - Delay: *apiTask.RestartPolicy.Delay, - Mode: *apiTask.RestartPolicy.Mode, + Attempts: *apiTask.RestartPolicy.Attempts, + Interval: *apiTask.RestartPolicy.Interval, + Delay: *apiTask.RestartPolicy.Delay, + Mode: *apiTask.RestartPolicy.Mode, + RenderTemplates: *apiTask.RestartPolicy.RenderTemplates, } } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index c49739cce8e1..df981735bb60 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -2518,10 +2518,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, }, RestartPolicy: &api.RestartPolicy{ - Interval: pointer.Of(1 * time.Second), - Attempts: pointer.Of(5), - Delay: pointer.Of(10 * time.Second), - Mode: pointer.Of("delay"), + Interval: pointer.Of(1 * time.Second), + Attempts: pointer.Of(5), + Delay: pointer.Of(10 * time.Second), + Mode: pointer.Of("delay"), + RenderTemplates: pointer.Of(false), }, ReschedulePolicy: &api.ReschedulePolicy{ Interval: pointer.Of(12 * time.Hour), @@ -2661,10 +2662,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, }, RestartPolicy: &api.RestartPolicy{ - Interval: pointer.Of(2 * time.Second), - Attempts: pointer.Of(10), - Delay: pointer.Of(20 * time.Second), - Mode: pointer.Of("delay"), + Interval: pointer.Of(2 * time.Second), + Attempts: pointer.Of(10), + Delay: pointer.Of(20 * time.Second), + Mode: pointer.Of("delay"), + RenderTemplates: pointer.Of(false), }, Services: []*api.Service{ { @@ -2927,10 +2929,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, }, RestartPolicy: &structs.RestartPolicy{ - Interval: 1 * time.Second, - Attempts: 5, - Delay: 10 * time.Second, - Mode: "delay", + Interval: 1 * time.Second, + Attempts: 5, + Delay: 10 * time.Second, + Mode: "delay", + RenderTemplates: false, }, Spreads: []*structs.Spread{ { @@ -3075,10 +3078,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, }, RestartPolicy: &structs.RestartPolicy{ - Interval: 2 * time.Second, - Attempts: 10, - Delay: 20 * time.Second, - Mode: "delay", + Interval: 2 * time.Second, + Attempts: 10, + Delay: 20 * time.Second, + Mode: "delay", + RenderTemplates: false, }, Services: []*structs.Service{ { @@ -3283,10 +3287,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, }, RestartPolicy: &api.RestartPolicy{ - Interval: pointer.Of(1 * time.Second), - Attempts: pointer.Of(5), - Delay: pointer.Of(10 * time.Second), - Mode: pointer.Of("delay"), + Interval: pointer.Of(1 * time.Second), + Attempts: pointer.Of(5), + Delay: pointer.Of(10 * time.Second), + Mode: pointer.Of("delay"), + RenderTemplates: pointer.Of(false), }, EphemeralDisk: &api.EphemeralDisk{ SizeMB: pointer.Of(100), @@ -3404,10 +3409,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, }, RestartPolicy: &structs.RestartPolicy{ - Interval: 1 * time.Second, - Attempts: 5, - Delay: 10 * time.Second, - Mode: "delay", + Interval: 1 * time.Second, + Attempts: 5, + Delay: 10 * time.Second, + Mode: "delay", + RenderTemplates: false, }, EphemeralDisk: &structs.EphemeralDisk{ SizeMB: 100, @@ -3462,10 +3468,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, }, RestartPolicy: &structs.RestartPolicy{ - Interval: 1 * time.Second, - Attempts: 5, - Delay: 10 * time.Second, - Mode: "delay", + Interval: 1 * time.Second, + Attempts: 5, + Delay: 10 * time.Second, + Mode: "delay", + RenderTemplates: false, }, Meta: map[string]string{ "lol": "code", diff --git a/command/agent/testingutils_test.go b/command/agent/testingutils_test.go index d1603be025ac..ab69d21f177b 100644 --- a/command/agent/testingutils_test.go +++ b/command/agent/testingutils_test.go @@ -35,10 +35,11 @@ func MockJob() *api.Job { SizeMB: pointer.Of(150), }, RestartPolicy: &api.RestartPolicy{ - Attempts: pointer.Of(3), - Interval: pointer.Of(10 * time.Minute), - Delay: pointer.Of(1 * time.Minute), - Mode: pointer.Of("delay"), + Attempts: pointer.Of(3), + Interval: pointer.Of(10 * time.Minute), + Delay: pointer.Of(1 * time.Minute), + Mode: pointer.Of("delay"), + RenderTemplates: pointer.Of(false), }, Networks: []*api.NetworkResource{ { diff --git a/command/helpers_test.go b/command/helpers_test.go index c5e83c1ff693..42948b592350 100644 --- a/command/helpers_test.go +++ b/command/helpers_test.go @@ -224,9 +224,10 @@ const ( resources {} } restart { - attempts = 10 - mode = "delay" - interval = "15s" + attempts = 10 + mode = "delay" + interval = "15s" + render_templates = false } } }` @@ -243,9 +244,10 @@ var ( Name: pointer.Of("group1"), Count: pointer.Of(1), RestartPolicy: &api.RestartPolicy{ - Attempts: pointer.Of(10), - Interval: pointer.Of(15 * time.Second), - Mode: pointer.Of("delay"), + Attempts: pointer.Of(10), + Interval: pointer.Of(15 * time.Second), + Mode: pointer.Of("delay"), + RenderTemplates: pointer.Of(false), }, Tasks: []*api.Task{ diff --git a/jobspec/parse_group.go b/jobspec/parse_group.go index 213e286a53fd..94dc3008d9c3 100644 --- a/jobspec/parse_group.go +++ b/jobspec/parse_group.go @@ -321,6 +321,7 @@ func parseRestartPolicy(final **api.RestartPolicy, list *ast.ObjectList) error { "interval", "delay", "mode", + "render_templates", } if err := checkHCLKeys(obj.Val, valid); err != nil { return err diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 3fe453c21a90..8a7342e415f9 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -193,10 +193,11 @@ func TestParse(t *testing.T) { "elb_checks": "3", }, RestartPolicy: &api.RestartPolicy{ - Interval: timeToPtr(10 * time.Minute), - Attempts: intToPtr(5), - Delay: timeToPtr(15 * time.Second), - Mode: stringToPtr("delay"), + Interval: timeToPtr(10 * time.Minute), + Attempts: intToPtr(5), + Delay: timeToPtr(15 * time.Second), + Mode: stringToPtr("delay"), + RenderTemplates: boolToPtr(false), }, Spreads: []*api.Spread{ { diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 66695c1c326f..aba8ea342b23 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -102,10 +102,11 @@ job "binstore-storagelocker" { } restart { - attempts = 5 - interval = "10m" - delay = "15s" - mode = "delay" + attempts = 5 + interval = "10m" + delay = "15s" + mode = "delay" + render_templates = false } reschedule { diff --git a/jobspec2/parse_test.go b/jobspec2/parse_test.go index a69eed7475fc..5fa6e77dfdc3 100644 --- a/jobspec2/parse_test.go +++ b/jobspec2/parse_test.go @@ -1086,3 +1086,21 @@ func TestErrMissingKey(t *testing.T) { require.NotNil(t, tmpl.ErrMissingKey) require.True(t, *tmpl.ErrMissingKey) } + +func TestRestartRenderTemplates(t *testing.T) { + ci.Parallel(t) + hclBytes, err := os.ReadFile("test-fixtures/restart-render-templates.hcl") + require.NoError(t, err) + job, err := ParseWithConfig(&ParseConfig{ + Path: "test-fixtures/restart-render-templates.hcl", + Body: hclBytes, + AllowFS: false, + }) + require.NoError(t, err) + tg := job.TaskGroups[0] + require.NotNil(t, tg.RestartPolicy) + require.True(t, *tg.RestartPolicy.RenderTemplates) + + require.Nil(t, tg.Tasks[0].RestartPolicy) + require.False(t, *tg.Tasks[1].RestartPolicy.RenderTemplates) +} diff --git a/jobspec2/test-fixtures/restart-render-templates.hcl b/jobspec2/test-fixtures/restart-render-templates.hcl new file mode 100644 index 000000000000..71e82c641987 --- /dev/null +++ b/jobspec2/test-fixtures/restart-render-templates.hcl @@ -0,0 +1,17 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +job "example" { + group "group" { + restart { + render_templates = true + } + task "foo" { + } + task "bar" { + restart { + render_templates = false + } + } + } +} \ No newline at end of file diff --git a/nomad/mock/job.go b/nomad/mock/job.go index 3e3d48c087ca..450f801ed5b0 100644 --- a/nomad/mock/job.go +++ b/nomad/mock/job.go @@ -44,10 +44,11 @@ func Job() *structs.Job { SizeMB: 150, }, RestartPolicy: &structs.RestartPolicy{ - Attempts: 3, - Interval: 10 * time.Minute, - Delay: 1 * time.Minute, - Mode: structs.RestartPolicyModeDelay, + Attempts: 3, + Interval: 10 * time.Minute, + Delay: 1 * time.Minute, + Mode: structs.RestartPolicyModeDelay, + RenderTemplates: false, }, ReschedulePolicy: &structs.ReschedulePolicy{ Attempts: 2, diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index 7d03c1fd48ad..f7286d69f4c1 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -1861,6 +1861,12 @@ func TestTaskGroupDiff(t *testing.T) { Old: "", New: "fail", }, + { + Type: DiffTypeAdded, + Name: "RenderTemplates", + Old: "", + New: "false", + }, }, }, }, @@ -1908,6 +1914,12 @@ func TestTaskGroupDiff(t *testing.T) { Old: "fail", New: "", }, + { + Type: DiffTypeDeleted, + Name: "RenderTemplates", + Old: "false", + New: "", + }, }, }, }, @@ -1925,10 +1937,11 @@ func TestTaskGroupDiff(t *testing.T) { }, New: &TaskGroup{ RestartPolicy: &RestartPolicy{ - Attempts: 2, - Interval: 2 * time.Second, - Delay: 2 * time.Second, - Mode: "delay", + Attempts: 2, + Interval: 2 * time.Second, + Delay: 2 * time.Second, + Mode: "delay", + RenderTemplates: true, }, }, Expected: &TaskGroupDiff{ @@ -1962,6 +1975,12 @@ func TestTaskGroupDiff(t *testing.T) { Old: "fail", New: "delay", }, + { + Type: DiffTypeEdited, + Name: "RenderTemplates", + Old: "false", + New: "true", + }, }, }, }, @@ -1972,18 +1991,20 @@ func TestTaskGroupDiff(t *testing.T) { Contextual: true, Old: &TaskGroup{ RestartPolicy: &RestartPolicy{ - Attempts: 1, - Interval: 1 * time.Second, - Delay: 1 * time.Second, - Mode: "fail", + Attempts: 1, + Interval: 1 * time.Second, + Delay: 1 * time.Second, + Mode: "fail", + RenderTemplates: false, }, }, New: &TaskGroup{ RestartPolicy: &RestartPolicy{ - Attempts: 2, - Interval: 2 * time.Second, - Delay: 1 * time.Second, - Mode: "fail", + Attempts: 2, + Interval: 2 * time.Second, + Delay: 1 * time.Second, + Mode: "fail", + RenderTemplates: true, }, }, Expected: &TaskGroupDiff{ @@ -2017,6 +2038,12 @@ func TestTaskGroupDiff(t *testing.T) { Old: "fail", New: "fail", }, + { + Type: DiffTypeEdited, + Name: "RenderTemplates", + Old: "false", + New: "true", + }, }, }, }, diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index ae87429491fd..f912ee82bf20 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -5865,16 +5865,18 @@ var ( // Canonicalize in api/tasks.go DefaultServiceJobRestartPolicy = RestartPolicy{ - Delay: 15 * time.Second, - Attempts: 2, - Interval: 30 * time.Minute, - Mode: RestartPolicyModeFail, + Delay: 15 * time.Second, + Attempts: 2, + Interval: 30 * time.Minute, + Mode: RestartPolicyModeFail, + RenderTemplates: false, } DefaultBatchJobRestartPolicy = RestartPolicy{ - Delay: 15 * time.Second, - Attempts: 3, - Interval: 24 * time.Hour, - Mode: RestartPolicyModeFail, + Delay: 15 * time.Second, + Attempts: 3, + Interval: 24 * time.Hour, + Mode: RestartPolicyModeFail, + RenderTemplates: false, } ) @@ -6207,6 +6209,9 @@ type RestartPolicy struct { // Mode controls what happens when the task restarts more than attempt times // in an interval. Mode string + + // RenderTemplates is flag to explicitly render all templates on task restart + RenderTemplates bool } func (r *RestartPolicy) Copy() *RestartPolicy { diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 1053d821a597..c49a1041279b 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -4307,9 +4307,10 @@ func TestRestartPolicy_Validate(t *testing.T) { // Policy with acceptable restart options passes p := &RestartPolicy{ - Mode: RestartPolicyModeFail, - Attempts: 0, - Interval: 5 * time.Second, + Mode: RestartPolicyModeFail, + Attempts: 0, + Interval: 5 * time.Second, + RenderTemplates: true, } if err := p.Validate(); err != nil { t.Fatalf("err: %v", err) diff --git a/scheduler/util.go b/scheduler/util.go index ebf3bbce30bd..9dc0cf76f1e0 100644 --- a/scheduler/util.go +++ b/scheduler/util.go @@ -258,6 +258,12 @@ func tasksUpdated(jobA, jobB *structs.Job, taskGroup string) comparison { return difference("volume request", a.Volumes, b.Volumes) } + // Check if restart.render_templates is updated + // this requires a destructive update for template hook to receive the new config + if a.RestartPolicy.RenderTemplates != b.RestartPolicy.RenderTemplates { + return difference("group restart render_templates", a.RestartPolicy.RenderTemplates, b.RestartPolicy.RenderTemplates) + } + // Check each task for _, at := range a.Tasks { bt := b.LookupTask(at.Name) @@ -319,6 +325,11 @@ func tasksUpdated(jobA, jobB *structs.Job, taskGroup string) comparison { if at.LogConfig.Disabled != bt.LogConfig.Disabled { return difference("task log disabled", at.LogConfig.Disabled, bt.LogConfig.Disabled) } + + // Check if restart.render_templates is updated + if at.RestartPolicy.RenderTemplates != bt.RestartPolicy.RenderTemplates { + return difference("task restart render_templates", at.RestartPolicy.RenderTemplates, bt.RestartPolicy.RenderTemplates) + } } // none of the fields that trigger a destructive update were modified, diff --git a/scheduler/util_test.go b/scheduler/util_test.go index 947e554781da..e3f578ece568 100644 --- a/scheduler/util_test.go +++ b/scheduler/util_test.go @@ -1407,3 +1407,19 @@ func TestUtil_UpdateNonTerminalAllocsToLost(t *testing.T) { expected = []string{} require.True(t, reflect.DeepEqual(allocsLost, expected), "actual: %v, expected: %v", allocsLost, expected) } + +func TestTaskGroupUpdated_Restart(t *testing.T) { + ci.Parallel(t) + + j1 := mock.Job() + name := j1.TaskGroups[0].Name + j2 := j1.Copy() + j3 := j1.Copy() + + must.False(t, tasksUpdated(j1, j2, name).modified) + j2.TaskGroups[0].RestartPolicy.RenderTemplates = true + must.True(t, tasksUpdated(j1, j2, name).modified) + + j3.TaskGroups[0].Tasks[0].RestartPolicy.RenderTemplates = true + must.True(t, tasksUpdated(j1, j3, name).modified) +} diff --git a/website/content/docs/job-specification/restart.mdx b/website/content/docs/job-specification/restart.mdx index 683254e7fdc3..5b0883665781 100644 --- a/website/content/docs/job-specification/restart.mdx +++ b/website/content/docs/job-specification/restart.mdx @@ -36,10 +36,11 @@ For example, assuming that the task group restart policy is: ```hcl restart { - interval = "30m" - attempts = 2 - delay = "15s" - mode = "fail" + interval = "30m" + attempts = 2 + delay = "15s" + mode = "fail" + render_templates = true } ``` @@ -55,10 +56,11 @@ then the effective restart policy for the task will be: ```hcl restart { - interval = "30m" - attempts = 5 - delay = "15s" - mode = "fail" + interval = "30m" + attempts = 5 + delay = "15s" + mode = "fail" + render_templates = true } ``` @@ -86,6 +88,11 @@ level, so that the Connect sidecar can inherit the default `restart`. than `attempts` times in an interval. For a detailed explanation of these values and their behavior, please see the [mode values section](#mode-values). +- `render_templates` `(bool: false)` - Specifies whether to re-render all +templates when a task is restarted. If set to `true`, all templates will be re-rendered +when the task restarts. This can be useful for re-fetching Vault secrets, even if the +lease on the existing secrets has not yet expired. + ### `restart` Parameter Defaults The values for many of the `restart` parameters vary by job type. Here are the @@ -95,10 +102,11 @@ defaults by job type: ```hcl restart { - attempts = 3 - delay = "15s" - interval = "24h" - mode = "fail" + attempts = 3 + delay = "15s" + interval = "24h" + mode = "fail" + render_templates = false } ``` @@ -106,10 +114,11 @@ defaults by job type: ```hcl restart { - interval = "30m" - attempts = 2 - delay = "15s" - mode = "fail" + interval = "30m" + attempts = 2 + delay = "15s" + mode = "fail" + render_templates = false } ```