Skip to content

Commit

Permalink
feature: Add new field render_templates on restart block (#18054)
Browse files Browse the repository at this point in the history 
This feature is necessary when user want to explicitly re-render all templates on task restart.
E.g. to fetch all new secrets from Vault, even if the lease on the existing secrets has not been expired.
  • Loading branch information
@nvanthao
nvanthao authored Jul 28, 2023
1 parent b17c0f7 commit 9e98d69
Show file tree
Hide file tree
Showing 21 changed files with 314 additions and 166 deletions.
3 changes: 3 additions & 0 deletions .changelog/18054.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:improvement
jobspec: Add new parameter `render_templates` for `restart` block to allow explicit re-render of templates on task restart. The default value is `false` and is fully backward compatible
```
106 changes: 58 additions & 48 deletions api/jobs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,10 +320,11 @@ func TestJobs_Canonicalize(t *testing.T) {
SizeMB: pointerOf(300),
},
RestartPolicy: &RestartPolicy{
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
Expand Down Expand Up @@ -405,10 +406,11 @@ func TestJobs_Canonicalize(t *testing.T) {
SizeMB: pointerOf(300),
},
RestartPolicy: &RestartPolicy{
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(3),
Interval: pointerOf(24 * time.Hour),
Mode: pointerOf("fail"),
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(3),
Interval: pointerOf(24 * time.Hour),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(1),
Expand Down Expand Up @@ -495,10 +497,11 @@ func TestJobs_Canonicalize(t *testing.T) {
SizeMB: pointerOf(300),
},
RestartPolicy: &RestartPolicy{
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
Expand Down Expand Up @@ -663,10 +666,11 @@ func TestJobs_Canonicalize(t *testing.T) {
Name: pointerOf("cache"),
Count: pointerOf(1),
RestartPolicy: &RestartPolicy{
Interval: pointerOf(5 * time.Minute),
Attempts: pointerOf(10),
Delay: pointerOf(25 * time.Second),
Mode: pointerOf("delay"),
Interval: pointerOf(5 * time.Minute),
Attempts: pointerOf(10),
Delay: pointerOf(25 * time.Second),
Mode: pointerOf("delay"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
Expand Down Expand Up @@ -707,10 +711,11 @@ func TestJobs_Canonicalize(t *testing.T) {
}},
},
RestartPolicy: &RestartPolicy{
Interval: pointerOf(5 * time.Minute),
Attempts: pointerOf(20),
Delay: pointerOf(25 * time.Second),
Mode: pointerOf("delay"),
Interval: pointerOf(5 * time.Minute),
Attempts: pointerOf(20),
Delay: pointerOf(25 * time.Second),
Mode: pointerOf("delay"),
RenderTemplates: pointerOf(false),
},
Resources: &Resources{
CPU: pointerOf(500),
Expand Down Expand Up @@ -835,7 +840,6 @@ func TestJobs_Canonicalize(t *testing.T) {
},
},
},

{
name: "update_merge",
input: &Job{
Expand Down Expand Up @@ -928,10 +932,11 @@ func TestJobs_Canonicalize(t *testing.T) {
SizeMB: pointerOf(300),
},
RestartPolicy: &RestartPolicy{
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
Expand Down Expand Up @@ -975,10 +980,11 @@ func TestJobs_Canonicalize(t *testing.T) {
SizeMB: pointerOf(300),
},
RestartPolicy: &RestartPolicy{
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
Expand Down Expand Up @@ -1016,7 +1022,6 @@ func TestJobs_Canonicalize(t *testing.T) {
},
},
},

{
name: "restart_merge",
input: &Job{
Expand All @@ -1036,8 +1041,9 @@ func TestJobs_Canonicalize(t *testing.T) {
{
Name: "task1",
RestartPolicy: &RestartPolicy{
Attempts: pointerOf(5),
Delay: pointerOf(1 * time.Second),
Attempts: pointerOf(5),
Delay: pointerOf(1 * time.Second),
RenderTemplates: pointerOf(true),
},
},
},
Expand Down Expand Up @@ -1105,10 +1111,11 @@ func TestJobs_Canonicalize(t *testing.T) {
SizeMB: pointerOf(300),
},
RestartPolicy: &RestartPolicy{
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
Expand Down Expand Up @@ -1140,10 +1147,11 @@ func TestJobs_Canonicalize(t *testing.T) {
Resources: DefaultResources(),
KillTimeout: pointerOf(5 * time.Second),
RestartPolicy: &RestartPolicy{
Attempts: pointerOf(5),
Delay: pointerOf(1 * time.Second),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
Attempts: pointerOf(5),
Delay: pointerOf(1 * time.Second),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(true),
},
},
},
Expand All @@ -1157,10 +1165,11 @@ func TestJobs_Canonicalize(t *testing.T) {
SizeMB: pointerOf(300),
},
RestartPolicy: &RestartPolicy{
Delay: pointerOf(20 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
Delay: pointerOf(20 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
ReschedulePolicy: &ReschedulePolicy{
Attempts: pointerOf(0),
Expand Down Expand Up @@ -1192,10 +1201,11 @@ func TestJobs_Canonicalize(t *testing.T) {
Resources: DefaultResources(),
KillTimeout: pointerOf(5 * time.Second),
RestartPolicy: &RestartPolicy{
Delay: pointerOf(20 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
Delay: pointerOf(20 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf("fail"),
RenderTemplates: pointerOf(false),
},
},
},
Expand Down
30 changes: 18 additions & 12 deletions api/tasks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,11 @@ type AllocCheckStatuses map[string]AllocCheckStatus
// RestartPolicy defines how the Nomad client restarts
// tasks in a taskgroup when they fail
type RestartPolicy struct {
Interval *time.Duration `hcl:"interval,optional"`
Attempts *int `hcl:"attempts,optional"`
Delay *time.Duration `hcl:"delay,optional"`
Mode *string `hcl:"mode,optional"`
Interval *time.Duration `hcl:"interval,optional"`
Attempts *int `hcl:"attempts,optional"`
Delay *time.Duration `hcl:"delay,optional"`
Mode *string `hcl:"mode,optional"`
RenderTemplates *bool `mapstructure:"render_templates" hcl:"render_templates,optional"`
}

func (r *RestartPolicy) Merge(rp *RestartPolicy) {
Expand All @@ -107,6 +108,9 @@ func (r *RestartPolicy) Merge(rp *RestartPolicy) {
if rp.Mode != nil {
r.Mode = rp.Mode
}
if rp.RenderTemplates != nil {
r.RenderTemplates = rp.RenderTemplates
}
}

// Reschedule configures how Tasks are rescheduled when they crash or fail.
Expand Down Expand Up @@ -580,21 +584,23 @@ func (g *TaskGroup) Canonicalize(job *Job) {
// in nomad/structs/structs.go
func defaultServiceJobRestartPolicy() *RestartPolicy {
return &RestartPolicy{
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf(RestartPolicyModeFail),
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(2),
Interval: pointerOf(30 * time.Minute),
Mode: pointerOf(RestartPolicyModeFail),
RenderTemplates: pointerOf(false),
}
}

// These needs to be in sync with DefaultBatchJobRestartPolicy in
// in nomad/structs/structs.go
func defaultBatchJobRestartPolicy() *RestartPolicy {
return &RestartPolicy{
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(3),
Interval: pointerOf(24 * time.Hour),
Mode: pointerOf(RestartPolicyModeFail),
Delay: pointerOf(15 * time.Second),
Attempts: pointerOf(3),
Interval: pointerOf(24 * time.Hour),
Mode: pointerOf(RestartPolicyModeFail),
RenderTemplates: pointerOf(false),
}
}

Expand Down
17 changes: 9 additions & 8 deletions client/allocrunner/taskrunner/task_runner_hooks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,14 +111,15 @@ func (tr *TaskRunner) initHooks() {
// If there are templates is enabled, add the hook
if len(task.Templates) != 0 {
tr.runnerHooks = append(tr.runnerHooks, newTemplateHook(&templateHookConfig{
logger: hookLogger,
lifecycle: tr,
events: tr,
templates: task.Templates,
clientConfig: tr.clientConfig,
envBuilder: tr.envBuilder,
consulNamespace: consulNamespace,
nomadNamespace: tr.alloc.Job.Namespace,
logger: hookLogger,
lifecycle: tr,
events: tr,
templates: task.Templates,
clientConfig: tr.clientConfig,
envBuilder: tr.envBuilder,
consulNamespace: consulNamespace,
nomadNamespace: tr.alloc.Job.Namespace,
renderOnTaskRestart: task.RestartPolicy.RenderTemplates,
}))
}

Expand Down
10 changes: 9 additions & 1 deletion client/allocrunner/taskrunner/template_hook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ type templateHookConfig struct {

// nomadNamespace is the job's Nomad namespace
nomadNamespace string

// renderOnTaskRestart is flag to explicitly render templates on task restart
renderOnTaskRestart bool
}

type templateHook struct {
Expand Down Expand Up @@ -97,7 +100,12 @@ func (h *templateHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar

// If we have already run prerun before exit early.
if h.templateManager != nil {
return nil
if !h.config.renderOnTaskRestart {
return nil
}
h.logger.Info("re-rendering templates on task restart")
h.templateManager.Stop()
h.templateManager = nil
}

// Store the current Vault token and the task directory
Expand Down
18 changes: 10 additions & 8 deletions command/agent/job_endpoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1058,10 +1058,11 @@ func ApiTgToStructsTG(job *structs.Job, taskGroup *api.TaskGroup, tg *structs.Ta
tg.Consul = apiConsulToStructs(taskGroup.Consul)

tg.RestartPolicy = &structs.RestartPolicy{
Attempts: *taskGroup.RestartPolicy.Attempts,
Interval: *taskGroup.RestartPolicy.Interval,
Delay: *taskGroup.RestartPolicy.Delay,
Mode: *taskGroup.RestartPolicy.Mode,
Attempts: *taskGroup.RestartPolicy.Attempts,
Interval: *taskGroup.RestartPolicy.Interval,
Delay: *taskGroup.RestartPolicy.Delay,
Mode: *taskGroup.RestartPolicy.Mode,
RenderTemplates: *taskGroup.RestartPolicy.RenderTemplates,
}

if taskGroup.ShutdownDelay != nil {
Expand Down Expand Up @@ -1209,10 +1210,11 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup,

if apiTask.RestartPolicy != nil {
structsTask.RestartPolicy = &structs.RestartPolicy{
Attempts: *apiTask.RestartPolicy.Attempts,
Interval: *apiTask.RestartPolicy.Interval,
Delay: *apiTask.RestartPolicy.Delay,
Mode: *apiTask.RestartPolicy.Mode,
Attempts: *apiTask.RestartPolicy.Attempts,
Interval: *apiTask.RestartPolicy.Interval,
Delay: *apiTask.RestartPolicy.Delay,
Mode: *apiTask.RestartPolicy.Mode,
RenderTemplates: *apiTask.RestartPolicy.RenderTemplates,
}
}

Expand Down
Loading

0 comments on commit 9e98d69

Please sign in to comment.