From 29a4f68937a485eb6b48f2487bbf0c1fa11a0170 Mon Sep 17 00:00:00 2001 From: tyrannasaurusbanks Date: Mon, 2 Jul 2018 09:10:44 +0100 Subject: [PATCH] Add aliveHandler to simply verify controller livesness. Add liveness check into deployment.yaml. Add explicit responseCode to stateHandler. Also fix some compiler warnings in alb-controller and correct log statement in ec2. --- .../templates/controller-deployment.yaml | 8 +++++++ alb-ingress-controller-helm/values.yaml | 3 +++ cmd/main.go | 1 + pkg/aws/albec2/ec2.go | 2 +- pkg/controller/alb-controller.go | 19 +++++++++++++++ pkg/controller/alb-controller_test.go | 24 +++++++++++++++++++ 6 files changed, 56 insertions(+), 1 deletion(-) diff --git a/alb-ingress-controller-helm/templates/controller-deployment.yaml b/alb-ingress-controller-helm/templates/controller-deployment.yaml index 0b610f9fb..10349f966 100644 --- a/alb-ingress-controller-helm/templates/controller-deployment.yaml +++ b/alb-ingress-controller-helm/templates/controller-deployment.yaml @@ -65,6 +65,14 @@ spec: scheme: HTTP initialDelaySeconds: 30 periodSeconds: {{ .Values.controller.readinessProbeInterval }} + timeout: {{ .Values.controller.readinessProbeTimeout }} + livenessProbe: + httpGet: + path: /alive + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 60 {{- if .Values.controller.resources }} resources: {{ toYaml .Values.controller.resources | indent 12 }} diff --git a/alb-ingress-controller-helm/values.yaml b/alb-ingress-controller-helm/values.yaml index 12f190cde..74c6099ee 100644 --- a/alb-ingress-controller-helm/values.yaml +++ b/alb-ingress-controller-helm/values.yaml @@ -43,6 +43,9 @@ controller: # How often (in seconds) to check controller readiness readinessProbeInterval: 10 + # How long to wait before timeout (in seconds) when checking controller readiness + readinessProbeTimeout: 1 + resources: {} # limits: # cpu: 100m diff --git a/cmd/main.go b/cmd/main.go index 6a51e286a..37c878699 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -56,6 +56,7 @@ func main() { http.HandleFunc("/state", ac.StateHandler) http.HandleFunc("/healthz", ac.StatusHandler) + http.HandleFunc("/alive", ac.AliveHandler) defer func() { logger.Infof("Shutting down ingress controller...") diff --git a/pkg/aws/albec2/ec2.go b/pkg/aws/albec2/ec2.go index fd8549cc0..7214bb395 100644 --- a/pkg/aws/albec2/ec2.go +++ b/pkg/aws/albec2/ec2.go @@ -84,7 +84,7 @@ func (e *EC2) DescribeSGByPermissionGroup(sg *string) (*string, error) { } if len(o.SecurityGroups) != 1 { - return nil, fmt.Errorf("Found more than 1 matching (managed) instance SGs. Found %d", len(o.SecurityGroups)) + return nil, fmt.Errorf("Didn't find exactly 1 matching (managed) instance SG. Found %d", len(o.SecurityGroups)) } e.cache.Set(key, o.SecurityGroups[0].GroupId, time.Minute*5) diff --git a/pkg/controller/alb-controller.go b/pkg/controller/alb-controller.go index 2c5a0f439..23eaae40c 100644 --- a/pkg/controller/alb-controller.go +++ b/pkg/controller/alb-controller.go @@ -63,7 +63,10 @@ type albController struct { var logger *log.Logger +// Release contains a default value but it's also exported so that it can be overriden with buildFlags var Release = "1.0.0" + +// Build contains a default value but it's also exported so that it can be overriden with buildFlags var Build = "git-00000000" func init() { @@ -315,6 +318,7 @@ func (ac *albController) StateHandler(w http.ResponseWriter, r *http.Request) { ac.mutex.RLock() defer ac.mutex.RUnlock() w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) _ = json.NewEncoder(w).Encode(ac.ALBIngresses) } @@ -356,6 +360,21 @@ func (ac *albController) StatusHandler(w http.ResponseWriter, r *http.Request) { encoder.Encode(checkResults) } +// AliveHandler validates the bare-minimum internals and only returns a empty response. +// It checks nothing downstream & should only used to ensure the controller is still running. +func (ac *albController) AliveHandler(w http.ResponseWriter, r *http.Request) { + // Take a lock here as a lightweight/minimum way to check the controller is alive. + ac.mutex.RLock() + defer ac.mutex.RUnlock() + w.Header().Set("Content-Type", "application/json") + // Explicitly set a healthy response so that this handler can be used to ascertain liveness. + w.WriteHeader(http.StatusOK) + + // Kubernetes only cares about the HTTP status code, so just return an empty body + w.Write([]byte("{}\n")) + return +} + // UpdateIngressStatus returns the hostnames for the ALB. func (ac *albController) UpdateIngressStatus(ing *extensions.Ingress) []api.LoadBalancerIngress { id := albingress.GenerateID(ing.ObjectMeta.Namespace, ing.ObjectMeta.Name) diff --git a/pkg/controller/alb-controller_test.go b/pkg/controller/alb-controller_test.go index ea0fd42e4..d50f4688b 100644 --- a/pkg/controller/alb-controller_test.go +++ b/pkg/controller/alb-controller_test.go @@ -313,6 +313,7 @@ func TestALBController_StateHandler(t *testing.T) { ingress := albingress.ALBIngress{} encodedIngressByteSlice, _ := json.Marshal(ingress) expectedBody := fmt.Sprintf("[%s]\n", encodedIngressByteSlice) + expectedResponseCode := 200 ac := albController{ ALBIngresses: []*albingress.ALBIngress{&ingress}, } @@ -323,6 +324,9 @@ func TestALBController_StateHandler(t *testing.T) { if rw.Header().Get("Content-Type") != "application/json" { t.Errorf("Expected header Content-Type: application-json") } + if rw.Result().StatusCode != expectedResponseCode { + t.Errorf("Expected http status code to be %d, got %d", expectedResponseCode, rw.Result().StatusCode) + } bodyString := fmt.Sprintf("%s", rw.Body.Bytes()) if expectedBody != bodyString { t.Errorf("Expected response body to be '%s', found '%s'", expectedBody, bodyString) @@ -366,3 +370,23 @@ func TestALBController_StatusHandler(t *testing.T) { } } } +func TestALBController_AliveHandler(t *testing.T) { + + expectedResponseCode := 200 + expectedResponseBody := "{}\n" + ac := albController{} + w := httptest.NewRecorder() + ac.AliveHandler(w, nil) + + if w.Header().Get("Content-Type") != "application/json" { + t.Errorf("Expected header Content-Type: application-json") + } + if w.Result().StatusCode != expectedResponseCode { + t.Errorf("Expected http status code to be %d, got %d", expectedResponseCode, w.Result().StatusCode) + } + + responseBody := fmt.Sprintf("%s", string(w.Body.Bytes()[:w.Body.Len()])) + if responseBody != expectedResponseBody { + t.Errorf("Expected response body to be '%s', found '%s'", expectedResponseBody, responseBody) + } +}