diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d007172..0000000 --- a/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Alpine IMAGE=3.7 installs keepalived 1.3; use for older linux kernels -# Alpine IMAGE=3.8+ installs keepalived 2.0: use for newer linux kernels - -FROM alpine:3.8 -LABEL architecture="x86_64" \ - license="Apache 2" \ - name="bsctl/keepalived" \ - summary="Alpine based keepalived container" \ - mantainer="bsctl" - -ENV KEEPALIVED_INTERFACE=eth0 \ - KEEPALIVED_STATE=MASTER \ - KEEPALIVED_PASSWORD=cGFzc3dvcmQK \ - KEEPALIVED_HEALTH_SERVICE_NAME=pidof \ - KEEPALIVED_HEALTH_SERVICE_INTERVAL=10 \ - KEEPALIVED_HEALTH_SERVICE_TIMEOUT=1 \ - KEEPALIVED_HEALTH_SERVICE_CHECK="/bin/pidof keepalived" \ - KEEPALIVED_HEALTH_SERVICE_USER=root \ - KEEPALIVED_HEALTH_SERVICE_RISE=1 \ - KEEPALIVED_HEALTH_SERVICE_FALL=1 \ - KEEPALIVED_ROUTER_ID=100 \ - KEEPALIVED_VIRTUAL_IP=1.1.1.1 \ - KEEPALIVED_ADVERT_INT=1 \ - KEEPALIVED_UNICAST_PEER= - -RUN apk add --no-cache curl keepalived -COPY scripts/config.sh /usr/bin -COPY scripts/keepalived.sh /usr/bin -COPY template.conf /etc/keepalived/template.conf -RUN chmod +x /usr/bin/config.sh && chmod +x /usr/bin/keepalived.sh -ENTRYPOINT ["/usr/bin/keepalived.sh"] -CMD ["-nlPd"] \ No newline at end of file diff --git a/LICENSE b/LICENSE index 6d53549..69e990e 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2020 CLASTIX + Copyright [2020] Clastix Labs Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 42700b8..6230bcf 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,12 @@ -# kube-keepalived - Kubernetes Control Plane with keepalived +# Kubernetes Control Plane with keepalived + +

+ + + + +

+ In a typical kubernetes setup, we have worker nodes and master nodes. Master nodes run the api-server, the etcd database, and the controllers. These nodes implement the so-called Control Plane, i.e. the brain of your Kubernetes cluster. Worker nodes talk to the Master nodes using the Control Plane Endpoint, usually a hostname or an IP address which tells all the nodes how to reach the Control Plane. ## What problem does it solve? @@ -38,50 +46,30 @@ Quote from the [website](https://www.keepalived.org/): *Keepalived is a routing software written in C. The main goal of this project is to provide simple and robust facilities for load-balancing and high-availability to Linux system and Linux based infrastructures. Loadbalancing framework relies on a well-known and widely used Linux Virtual Server (IPVS) kernel module providing Layer4 load-balancing. Keepalived implements a set of checkers to dynamically and adaptively maintain and manage a load-balanced server pool according to their health.* -... - *On the other hand high-availability is achieved by VRRP protocol. VRRP is a fundamental brick for router failover. Also, Keepalived implements a set of hooks to the VRRP finite state machine providing low-level and high-speed protocol interactions. To offer faster network failure detection, Keepalived implements the BFD protocol. VRRP state transition can take into account the BFD hint to drive fast state transition. Keepalived frameworks can be used independently or all together to provide resilient infrastructures.* ## Running keepalived as container -Keepalived runs on most of the modern Linux distributions as `systemd` daemon. In this guide, we propose a containerized version of it, not only just for fun. We want to run `keepalived` as part of the Kubernetes Control Plane and having strictly coupled with it. In our implementation, the `keepalived` is managed by the `kubelet` daemon running on every Kubernetes node. It is deployed as a static pod along with the other components: the api-server, the etcd data store, the controller-manager, and the scheduler. +Keepalived runs on most of the modern Linux distributions as `systemd` daemon. In this guide, we propose a containerized version of it, not only just for fun. We want to run `keepalived` as part of the Kubernetes Control Plane and having strictly coupled with it. In our implementation, the `keepalived` is managed by the `kubelet` daemon running on every Kubernetes node. It is deployed as a static pod along with the other components: the `apiserver`, the `etcd` data store, the `controller-manager`, and the `scheduler`. -First, we need to place `keepalived` in a Docker image. There are many good examples out there. The following is our proposal: +First, we need to place `keepalived` in a Docker image: ```Dockerfile -# Alpine IMAGE=3.7 installs keepalived 1.3; use for older linux kernels -# Alpine IMAGE=3.8+ installs keepalived 2.0: use for newer linux kernels -FROM alpine:3.8 +FROM alpine:3.10 + LABEL architecture="x86_64" \ license="Apache 2" \ - name="bsctl/keepalived" \ + name="keepalived" \ summary="Alpine based keepalived container" \ mantainer="bsctl" -ENV KEEPALIVED_INTERFACE=eth0 \ - KEEPALIVED_STATE=MASTER \ - KEEPALIVED_PASSWORD=cGFzc3dvcmQK \ - KEEPALIVED_HEALTH_SERVICE_NAME=pidof \ - KEEPALIVED_HEALTH_SERVICE_INTERVAL=10 \ - KEEPALIVED_HEALTH_SERVICE_TIMEOUT=1 \ - KEEPALIVED_HEALTH_SERVICE_CHECK="/bin/pidof keepalived" \ - KEEPALIVED_HEALTH_SERVICE_USER=root \ - KEEPALIVED_ROUTER_ID=100 \ - KEEPALIVED_VIRTUAL_IP=1.1.1.1 \ - KEEPALIVED_ADVERT_INT=1 \ - KEEPALIVED_UNICAST_PEER= - RUN apk add --no-cache curl keepalived -COPY scripts/config.sh /usr/bin -COPY scripts/keepalived.sh /usr/bin -COPY template.conf /etc/keepalived/template.conf -RUN chmod +x /usr/bin/config.sh && chmod +x /usr/bin/keepalived.sh -ENTRYPOINT ["/usr/bin/keepalived.sh"] -CMD ["-nlPd"] -``` +ENTRYPOINT ["/usr/sbin/keepalived","--dont-fork","--log-console"] -Since keepalived is compiled against specific versions of the Linux kernel, it is very important to have the Docker image aligned to the kernel running on your machines. For older kernels, e.g. 3.10, running on CentOS7 distribution, we build the image with Alpine 3.7 which provides keepalive 1.3 version. On the other side, for newer kernels, we build with Alpine 3.8+ as it uses more recent keepalive versions. +# Customise keepalived with: +# CMD ["--vrrp","--log-detail","--dump-conf"] +``` -The script [config.sh](./scripts/config.sh) just fills the configuration starting from a [placeholder](./template.conf) and the script [keepalived.sh](./scripts/keepalived.sh) takes care of starting the keepalived binary and pass it the arguments. +Since keepalived is compiled against specific versions of the Linux kernel, it is very important to have the Docker image aligned to the kernel running on your machines. For older kernels, e.g. 3.10, running in the CentOS 7 distribution, we build the image with Alpine 3.7 which provides `keepalived` 1.3 version. On the other side, for newer kernels, we build with Alpine 3.8+ as it uses more recent `keepalived` versions. ## Keepalived basics There are plenty of tutorials out there explaining the keepalived basics. However, we suggest checking the official [documentation](https://www.keepalived.org/manpage.html). @@ -109,11 +97,9 @@ vrrp_instance VI_1 { {{ KEEPALIVED_HEALTH_SERVICE_NAME }} } - #unicast unicast_peer { - {{ KEEPALIVED_UNICAST_PEER }} + {{ KEEPALIVED_UNICAST_PEERS }} } - #unicast virtual_ipaddress { {{ KEEPALIVED_VIRTUAL_IP }} label {{ KEEPALIVED_INTERFACE }}:VIP @@ -123,7 +109,7 @@ vrrp_instance VI_1 { We start off by telling keepalived to communicate with its peers over KEEPALIVED_INTERFACE. We set the instance state option to KEEPALIVED_STATE. This is the initial value that keepalived will use until the daemon contacts the other instances and hold an election. During the election, the priority option KEEPALIVED_PRIORITY is used to decide which member is elected and the decision is simply based on which instance has the highest number. We can force the priority of the keepalived instance but, in our implementation, if this option is left unassigned, the latest octet of the node's IP address is used. -The KEEPALIVED_ROUTER_ID option should be shared by all the keepalive instances participating in the same multicast group. By default, keepalived advertises on the multicast group `224.0.0.18`. If multicast is not enabled on the network, alternatively, we can switch to unicast by defining a list of unicast peers specified into the KEEPALIVED_UNICAST_PEER option. +The KEEPALIVED_ROUTER_ID option should be shared by all the keepalive instances participating in the same multicast group. By default, keepalived advertises on the multicast group `224.0.0.18`. If multicast is not enabled on the network, alternatively, we can switch to unicast by defining a list of unicast peers specified into the KEEPALIVED_UNICAST_PEERS option. Also, we can set up a simple authentication based on KEEPALIVED_PASSWORD for our keepalived instances to communicate with one another. @@ -150,7 +136,7 @@ It's possible to specify the user running the script, along with the interval in The script can check anything we want. As soon as the tracking script returns another code than 0, the VRRP instance will change the state to FAULT, removes the VIP from the network interface, and stops sending VRRP advertisements. In our case, we will write a simple script checking the health status of the Kubernetes api-server. ## Keepalived as part of the control plane -We want keepalived deployed as part of the Kubernetes control plane running on the master nodes along with the other components: the api-server, the etcd database, and the controllers. In many Kubernetes distributions, including Red Hat OpenShift, these components are running as privileged containers and deployed as static pods. +We want keepalived deployed as part of the Kubernetes control plane running on the master nodes along with the other components: the apiserver, the etcd database, and the controllers. In many Kubernetes distributions, including Red Hat OpenShift, these components are running as privileged containers and deployed as static pods. ### Static pods Static pods are managed directly by the `kubelet` daemon running on the node, without the api-server involvement. Unlike regular pods that are managed by the control plane instead, the kubelet watches static pods and restarts them when they crash. @@ -182,68 +168,61 @@ Static pods are created by the kubelet by watching a specific directory on the n The following [kube-keepalived.yaml](./kube-keepalived.yaml) is the manifest file we use to deploy a keepalived static pod on every master node: ```yaml +# Kubernetes: manifests for static pods are in /etc/kubernetes/manifests apiVersion: v1 kind: Pod metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - name: kube-keepalived - namespace: kube-system +annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" +labels: + app.kubernetes.io/name: kubelived + app.kubernetes.io/instance: kubelived +name: kube-keepalived +namespace: kube-system spec: - containers: - - name: keepalived - image: docker.io/bsctl/keepalived:2.0 - args: # override the CMD ["-nlPd"] in Dockerfile - - -nlPdD - env: - - name: KEEPALIVED_INTERFACE - value: 'eth0' - - name: KEEPALIVED_STATE - value: 'BACKUP' - - name: KEEPALIVED_PASSWORD # echo 'password' | base64 - value: 'cGFzc3dvcmQK' - - name: KEEPALIVED_HEALTH_SERVICE_NAME - value: 'apiserver' - - name: KEEPALIVED_HEALTH_SERVICE_CHECK - value: '/usr/bin/curl -s -k https://localhost:6443/healthz -o /dev/null' - - name: KEEPALIVED_HEALTH_SERVICE_USER - value: 'root' - - name: KEEPALIVED_ROUTER_ID - value: '100' - - name: KEEPALIVED_PRIORITY - value: '' - - name: KEEPALIVED_VIRTUAL_IP - value: '10.10.10.250' - - name: KEEPALIVED_ADVERT_INT - value: '3' - - name: KEEPALIVED_UNICAST_PEER - value: '10.10.10.10 10.10.10.11 10.10.10.12' - livenessProbe: - exec: - command: ["pidof", "keepalived"] - initialDelaySeconds: 10 - securityContext: - privileged: true - resources: - volumeMounts: - - mountPath: /etc/localtime - name: host-localtime - hostNetwork: true - priorityClassName: system-node-critical - restartPolicy: Always - volumes: - - hostPath: - path: /etc/localtime +containers: +- name: keepalived + image: bsctl/keepalived:0.2.0 + imagePullPolicy: Always + args: # override options in the Dockerfile + - --vrrp + - --log-detail + - --dump-conf + - --use-file=/etc/keepalived/keepalived.conf + livenessProbe: + exec: + command: ["pidof", "keepalived"] + initialDelaySeconds: 10 + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + resources: + volumeMounts: + - mountPath: /etc/localtime name: host-localtime + - mountPath: /etc/keepalived/keepalived.conf + name: config +hostNetwork: true +priorityClassName: system-node-critical +restartPolicy: Always +volumes: +- hostPath: + path: /etc/localtime + name: host-localtime +- hostPath: + path: /etc/keepalived/keepalived.conf + name: config ``` -The three master nodes have the following IP addresses assigned to their `eth0` network interface: +Suppose, we have three master nodes with following IP addresses assigned to their `eth0` network interface: * master-0: 10.10.10.10 * master-0: 10.10.10.11 * master-2: 10.10.10.12 -The floating VIP is 10.10.10.250 and it will be assigned to the master node with the highest priority. Since we are not passing the priority, it will be calculated as the last octet of the node's IPv4 address. According to the VRRP protocol, the VIP will be initially taken by the node with the highest priority, i.e. the master-2. +The floating VIP is 10.10.10.250 and it will be assigned to the master node with the highest priority. When not passing the priority, it can be calculated as the last octet of the node's IPv4 address. According to the VRRP protocol, the VIP will be initially taken by the node with the highest priority. As track script, we check the healthiness of the Kubernetes api-server as by polling its health URL: @@ -253,9 +232,25 @@ As track script, we check the healthiness of the Kubernetes api-server as by pol According to the above, the keepalive will move the VIP away from nodes having unhealth api-server. +### Deploying keepalived with Helm +Starting from release 0.2.0, we support the deployment of keepalived via the Helm 3 [chart](./helm). + +Make sure you have `kubectl` and `helm` tools installed in your workstation. Also make sure to have admin access to the cluster: + + +Install through Helm: -### Deploying keepalived -Copy the file above into `/etc/kubernetes/manifest` folder of every master node and check the mirror pods + $ helm install kubelived --namespace kube-system + +Actually, the Helm chart does not install `keepalived` on the Kubernetes Control Plane. Instead, it deployes installer pods as as daemonset on the master nodes. The installer pods are responsible to deploy the `kube-keepalived.yaml` pod manifest in the `/etc/kubernetes/manifests` folder and the proper `keepalived.conf` configuration file in `/etc/keepalived` location of each master node. + +Check the installer pods: + + $ kubectl -n kube-system get pods -o wide | grep keepalived-installer + +The installer pods do a lookup to find the proper keepalived interface starting from the VIP specified as `keepalived_virtual_address` parameter in the chart's [values.yaml](./helm/values.yaml). + +Check the installation of `keepalived`: $ kubectl -n kube-system get pods @@ -276,7 +271,7 @@ Copy the file above into `/etc/kubernetes/manifest` folder of every master node kube-keepalived-master-1 1/1 Running 1 1m kube-keepalived-master-2 1/1 Running 1 1m -Check the actual keepalived configuration according to the values we passed as environment variables. For example, the keepalive instance running on master-2 node has: +Check the actual keepalived configuration according to the values we passed in the chart: $ kubectl -n kube-system exec kube-keepalived-master-2 -- cat /etc/keepalived/keepalived.conf @@ -308,14 +303,6 @@ Check the actual keepalived configuration according to the values we passed as e track_script { apiserver } - - #unicast - unicast_peer { - 10.10.10.10 - 10.10.10.11 - 10.10.10.12 - } - #unicast virtual_ipaddress { 10.10.10.250 label eth0:VIP @@ -417,9 +404,4 @@ Sending USR1 signal to keepalive process will dump configuration data to `/tmp/k Received: 0 Sent: 0 -Also, SNMP V2 and V3 MIBs are available to monitor keepalived. - -## Deploy on Red Hat OpenShift -OpenShift is a PaaS built on top of [OKD](https://docs.okd.io/), the distribution of Kubernetes mantained by Red Hat. In order to make our solution working on Openshift, we need to adapt it a bit. In OpenShift, the manifest files for static pods are placed under `/etc/origin/node/pods` of the nodes, so we have to place the manifest of keepalived in this folder. - -In Openshift, the api-server is listening by default on the 8443 HTTPS port. The track script must be adapted accordly: `/usr/bin/curl -s -k https://localhost:8443/healthz -o /dev/null`. Here the manifest [kube-keepalived-okd.yaml](./kube-keepalived-okd.yaml) file for OpenShift. \ No newline at end of file +Also, SNMP V2 and V3 MIBs are available to monitor keepalived. \ No newline at end of file diff --git a/helm/.helmignore b/helm/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/Chart.yaml b/helm/Chart.yaml new file mode 100644 index 0000000..3a30615 --- /dev/null +++ b/helm/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: kubelived +description: A Helm chart for installing Keepalived for Kubernetes Control Plane + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 0.2.0 diff --git a/helm/templates/NOTES.txt b/helm/templates/NOTES.txt new file mode 100644 index 0000000..bb6c14a --- /dev/null +++ b/helm/templates/NOTES.txt @@ -0,0 +1,19 @@ +- Kubelived Helm Chart deployed: + + # Check the installer logs + $ kubectl logs -f ds/{{ template "kubelived.fullname" . }}-installer -n {{ .Release.Namespace }} + + # Check the keepalived logs + $ kubectl logs -f kube-keepalived- -n {{ .Release.Namespace }} + +- Manage this chart: + + # Upgrade Kubelived + $ helm upgrade {{ .Release.Name }} -f kubelived -n {{ .Release.Namespace }} + + # Show this status again + $ helm status {{ .Release.Name }} -n {{ .Release.Namespace }} + + # Uninstall Kubelived + $ helm uninstall {{ .Release.Name }} -n {{ .Release.Namespace }} + diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl new file mode 100644 index 0000000..78940a5 --- /dev/null +++ b/helm/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "kubelived.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kubelived.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kubelived.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "kubelived.labels" -}} +helm.sh/chart: {{ include "kubelived.chart" . }} +{{ include "kubelived.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kubelived.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kubelived.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kubelived.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "kubelived.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm/templates/keepalived-config.yaml b/helm/templates/keepalived-config.yaml new file mode 100644 index 0000000..ce6171f --- /dev/null +++ b/helm/templates/keepalived-config.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: keepalived-config + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelived.labels" . | nindent 4 }} +data: + {{ .Values.keepalived_config_file }}: | + # keepalived global configuration + global_defs { + default_interface KEEPALIVED_INTERFACE + enable_script_security + } + vrrp_script {{ .Values.keepalived_health_service_name }} { + script {{ quote .Values.keepalived_health_service_check }} + interval {{ .Values.keepalived_health_service_interval }} + timeout {{ .Values.keepalived_health_service_timeout }} + rise {{ .Values.keepalived_health_service_rise }} + fall {{ .Values.keepalived_health_service_fall }} + user {{ .Values.keepalived_health_service_user }} + } + vrrp_instance VI_1 { + state {{ upper .Values.keepalived_initial_state }} + interface KEEPALIVED_INTERFACE + virtual_router_id {{ .Values.keepalived_router_id }} + priority KEEPALIVED_PRIORITY + advert_int {{ .Values.keepalived_health_service_interval }} + authentication { + auth_type PASS + auth_pass {{ .Values.keepalived_password | b64enc }} + } + track_script { + {{ .Values.keepalived_health_service_name }} + } + virtual_ipaddress { + {{ .Values.keepalived_virtual_address }} label KEEPALIVED_INTERFACE:VIP + } + + {{- if .Values.keepalived_unicast_peers }} + unicast_peer { + {{- range .Values.keepalived_unicast_peers }} + {{ . }} + {{- end }} + } + {{- end }} + } \ No newline at end of file diff --git a/helm/templates/keepalived-install-scripts.yaml b/helm/templates/keepalived-install-scripts.yaml new file mode 100644 index 0000000..0898b3e --- /dev/null +++ b/helm/templates/keepalived-install-scripts.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: keepalived-install-scripts + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelived.labels" . | nindent 4 }} +data: + install.sh: | + #!/bin/sh + # + echo $(date): "Copying keepalived manifest file to /etc/kubernetes/manifests/kube-keepalived.yaml" + cp -f /opt/kube-keepalived.yaml /etc/kubernetes/manifests/kube-keepalived.yaml + chmod 600 /etc/kubernetes/manifests/kube-keepalived.yaml + # + echo $(date): "Copying keepalived config file to {{ .Values.keepalived_config_path }}/{{ .Values.keepalived_config_file }}" + cp -f /opt/{{ .Values.keepalived_config_file }} {{ .Values.keepalived_config_path }}/{{ .Values.keepalived_config_file }} + + KEEPALIVED_INTERFACE=$(ip route get {{ .Values.keepalived_virtual_address }} | awk '{print $3}') + echo $(date): "Finding the keepalived interface:" $KEEPALIVED_INTERFACE + KEEPALIVED_PRIORITY=$(ip route get {{ .Values.keepalived_virtual_address }} | awk '{print $5}' | cut -d. -f4) + echo $(date): "Calculating the keepalived priority:" $KEEPALIVED_PRIORITY + + sed -i "s|KEEPALIVED_INTERFACE|$KEEPALIVED_INTERFACE|g" {{ .Values.keepalived_config_path }}/{{ .Values.keepalived_config_file }} + sed -i "s|KEEPALIVED_PRIORITY|$KEEPALIVED_PRIORITY|g" {{ .Values.keepalived_config_path }}/{{ .Values.keepalived_config_file }} + echo $(date): "Updated the keepalived config file" + + # Unless told otherwise, sleep forever. + # This prevents Kubernetes from restarting the pod repeatedly. + should_sleep=true + echo $(date): "Done installing keepalived." + while [ "$should_sleep" = "true" ]; do + if [ -e {{ .Values.keepalived_config_path }}/{{ .Values.keepalived_config_file }} ]; + then + echo $(date): "Nothing to do, going to sleep again ..." + sleep 120 + else + should_sleep=false + echo $(date): "Missing keepalived configuration file: {{ .Values.keepalived_config_path }}/{{ .Values.keepalived_config_file }}" + fi + done + uninstall.sh: | + #!/bin/sh + # + echo $(date): "Removing keepalived manifest file from /etc/kubernetes/manifests/kube-keepalived.yaml" + rm -rf /etc/kubernetes/manifests/kube-keepalived.yaml + echo $(date): "Removing keepalived config file from {{ .Values.keepalived_config_path }}/{{ .Values.keepalived_config_file }}" + rm -rf {{ .Values.keepalived_config_path }}/{{ .Values.keepalived_config_file }} \ No newline at end of file diff --git a/helm/templates/keepalived-installer.yaml b/helm/templates/keepalived-installer.yaml new file mode 100644 index 0000000..a0531dd --- /dev/null +++ b/helm/templates/keepalived-installer.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: keepalived-installer + labels: + {{- include "kubelived.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "kubelived.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + {{- include "kubelived.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "kubelived.serviceAccountName" . }} + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + containers: + - name: installer + securityContext: + privileged: true + image: busybox + imagePullPolicy: IfNotPresent + command: ["./scripts/install.sh"] + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "./scripts/uninstall.sh"] + volumeMounts: + - name: manifest + mountPath: /etc/kubernetes/manifests + - name: config + mountPath: {{ .Values.keepalived_config_path }} + - name: keepalived-manifest + mountPath: /opt/kube-keepalived.yaml + subPath: kube-keepalived.yaml + - name: keepalived-config + mountPath: /opt/{{ .Values.keepalived_config_file }} + subPath: {{ .Values.keepalived_config_file }} + - name: keepalived-install-scripts + mountPath: /scripts + volumes: + - name: manifest + hostPath: + path: /etc/kubernetes/manifests + - name: config + hostPath: + path: {{ .Values.keepalived_config_path }} + type: DirectoryOrCreate + - name: keepalived-manifest + configMap: + name: keepalived-manifest + - name: keepalived-config + configMap: + name: keepalived-config + - name: keepalived-install-scripts + configMap: + name: keepalived-install-scripts + defaultMode: 0700 \ No newline at end of file diff --git a/helm/templates/keepalived-manifest.yaml b/helm/templates/keepalived-manifest.yaml new file mode 100644 index 0000000..444d3ec --- /dev/null +++ b/helm/templates/keepalived-manifest.yaml @@ -0,0 +1,54 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: keepalived-manifest + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelived.labels" . | nindent 4 }} +data: + kube-keepalived.yaml: | + # Kubernetes: manifest for static pods are in /etc/kubernetes/manifest + apiVersion: v1 + kind: Pod + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + {{- include "kubelived.selectorLabels" . | nindent 8 }} + name: kube-keepalived + namespace: kube-system + spec: + containers: + - name: keepalived + image: {{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }} + imagePullPolicy: Always + args: # override options in the Dockerfile + - --vrrp + - --log-detail + - --dump-conf + - --use-file={{ .Values.keepalived_config_path }}/{{ .Values.keepalived_config_file }} + livenessProbe: + exec: + command: ["pidof", "keepalived"] + initialDelaySeconds: 10 + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + resources: + volumeMounts: + - mountPath: /etc/localtime + name: host-localtime + - mountPath: {{ .Values.keepalived_config_path }}/{{ .Values.keepalived_config_file }} + name: config + hostNetwork: true + priorityClassName: system-node-critical + restartPolicy: Always + volumes: + - hostPath: + path: /etc/localtime + name: host-localtime + - hostPath: + path: {{ .Values.keepalived_config_path }}/{{ .Values.keepalived_config_file }} + name: config diff --git a/helm/templates/serviceaccount.yaml b/helm/templates/serviceaccount.yaml new file mode 100644 index 0000000..0a6f1ea --- /dev/null +++ b/helm/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubelived.serviceAccountName" . }} + labels: + {{- include "kubelived.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/values.yaml b/helm/values.yaml new file mode 100644 index 0000000..db7fa65 --- /dev/null +++ b/helm/values.yaml @@ -0,0 +1,29 @@ +# Default values for kubelived. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: bsctl/keepalived + pullPolicy: IfNotPresent + tag: '' + +serviceAccount: + create: false + annotations: {} + name: '' + +keepalived_config_path: '/etc/keepalived' +keepalived_config_file: 'keepalived.conf' +keepalived_initial_state: 'backup' +keepalived_password: 'password' +keepalived_health_service_name: 'apiserver' +keepalived_health_service_check: '/usr/bin/curl -s -k https://localhost:6443/healthz -o /dev/null' +keepalived_health_service_user: 'root' +keepalived_health_service_interval: '20' +keepalived_health_service_timeout: '5' +keepalived_health_service_rise: '1' +keepalived_health_service_fall: '1' +keepalived_router_id: '100' +keepalived_advertisement_interval: '3' +keepalived_virtual_address: '10.10.10.250' +keepalived_unicast_peers: [] diff --git a/keepalived/Dockerfile b/keepalived/Dockerfile new file mode 100644 index 0000000..0896117 --- /dev/null +++ b/keepalived/Dockerfile @@ -0,0 +1,13 @@ +FROM alpine:3.10 + +LABEL architecture="x86_64" \ + license="Apache 2" \ + name="keepalived" \ + summary="Alpine based keepalived container" \ + mantainer="bsctl" + +RUN apk add --no-cache curl keepalived +ENTRYPOINT ["/usr/sbin/keepalived","--dont-fork","--log-console"] + +# Customise keepalived with: +# CMD ["--vrrp","--log-detail","--dump-conf"] diff --git a/kube-keepalived-okd.yaml b/kube-keepalived-okd.yaml deleted file mode 100644 index b6474d1..0000000 --- a/kube-keepalived-okd.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# Openshift: manifest for static pods are in /etc/origin/node/pods -apiVersion: v1 -kind: Pod -metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - labels: - openshift.io/control-plane: 'true' - name: kube-keepalived - namespace: kube-system -spec: - containers: - - name: keepalived - image: docker.io/bsctl/keepalived:1.3 - args: # override the CMD ["-nlPd"] in Dockerfile - - -nlPdD - env: - - name: KEEPALIVED_INTERFACE - value: 'eth0' - - name: KEEPALIVED_STATE - value: 'BACKUP' - - name: KEEPALIVED_PASSWORD # echo 'password' | base64 - value: 'cGFzc3dvcmQK' - - name: KEEPALIVED_HEALTH_SERVICE_NAME - value: 'apiserver' - - name: KEEPALIVED_HEALTH_SERVICE_INTERVAL - value: '20' - - name: KEEPALIVED_HEALTH_SERVICE_TIMEOUT - value: '5' - - name: KEEPALIVED_HEALTH_SERVICE_CHECK - value: '/usr/bin/curl -s -k https://localhost:8443/healthz -o /dev/null' - - name: KEEPALIVED_HEALTH_SERVICE_USER - value: 'root' - - name: KEEPALIVED_ROUTER_ID - value: '100' - - name: KEEPALIVED_PRIORITY - value: '' - - name: KEEPALIVED_VIRTUAL_IP - value: '10.10.10.250' - - name: KEEPALIVED_ADVERT_INT - value: '3' - - name: KEEPALIVED_UNICAST_PEER - value: '10.10.10.10 10.10.10.11 10.10.10.12' - livenessProbe: - exec: - command: ["pidof", "keepalived"] - initialDelaySeconds: 10 - securityContext: - privileged: true - resources: - volumeMounts: - - mountPath: /etc/localtime - name: host-localtime - hostNetwork: true - priorityClassName: system-node-critical - restartPolicy: Always - volumes: - - hostPath: - path: /etc/localtime - name: host-localtime diff --git a/kube-keepalived.yaml b/kube-keepalived.yaml deleted file mode 100644 index 105b9bc..0000000 --- a/kube-keepalived.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# Kubernetes: manifest for static pods are in /etc/kubernetes/manifest -apiVersion: v1 -kind: Pod -metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - labels: - name: kube-keepalived - namespace: kube-system -spec: - containers: - - name: keepalived - image: docker.io/bsctl/keepalived:2.0 - args: # override the CMD ["-nlPd"] in Dockerfile - - -nlPdD - env: - - name: KEEPALIVED_INTERFACE - value: 'eth0' - - name: KEEPALIVED_STATE - value: 'BACKUP' - - name: KEEPALIVED_PASSWORD # echo 'password' | base64 - value: 'cGFzc3dvcmQK' - - name: KEEPALIVED_HEALTH_SERVICE_NAME - value: 'apiserver' - - name: KEEPALIVED_HEALTH_SERVICE_INTERVAL - value: '20' - - name: KEEPALIVED_HEALTH_SERVICE_TIMEOUT - value: '5' - - name: KEEPALIVED_HEALTH_SERVICE_CHECK - value: '/usr/bin/curl -s -k https://localhost:6443/healthz -o /dev/null' - - name: KEEPALIVED_HEALTH_SERVICE_USER - value: 'root' - - name: KEEPALIVED_ROUTER_ID - value: '100' - - name: KEEPALIVED_PRIORITY - value: '' - - name: KEEPALIVED_VIRTUAL_IP - value: '10.10.10.250' - - name: KEEPALIVED_ADVERT_INT - value: '3' - - name: KEEPALIVED_UNICAST_PEER - value: '10.10.10.10 10.10.10.11 10.10.10.12' - livenessProbe: - exec: - command: ["pidof", "keepalived"] - initialDelaySeconds: 10 - securityContext: - privileged: true - resources: - volumeMounts: - - mountPath: /etc/localtime - name: host-localtime - hostNetwork: true - priorityClassName: system-node-critical - restartPolicy: Always - volumes: - - hostPath: - path: /etc/localtime - name: host-localtime diff --git a/template.conf b/template.conf deleted file mode 100644 index c0bf372..0000000 --- a/template.conf +++ /dev/null @@ -1,39 +0,0 @@ -global_defs { - default_interface {{ KEEPALIVED_INTERFACE }} - enable_script_security -} - -vrrp_script {{ KEEPALIVED_HEALTH_SERVICE_NAME }} { - script "{{ KEEPALIVED_HEALTH_SERVICE_CHECK }}" - interval {{ KEEPALIVED_HEALTH_SERVICE_INTERVAL }} - timeout {{ KEEPALIVED_HEALTH_SERVICE_TIMEOUT }} - rise {{ KEEPALIVED_HEALTH_SERVICE_RISE }} - fall {{ KEEPALIVED_HEALTH_SERVICE_FALL }} - user {{ KEEPALIVED_HEALTH_SERVICE_USER }} -} - -vrrp_instance VI_1 { - state {{ KEEPALIVED_STATE }} - interface {{ KEEPALIVED_INTERFACE }} - virtual_router_id {{ KEEPALIVED_ROUTER_ID }} - priority {{ KEEPALIVED_PRIORITY }} - advert_int {{ KEEPALIVED_ADVERT_INT }} - authentication { - auth_type PASS - auth_pass {{ KEEPALIVED_PASSWORD }} - } - - track_script { - {{ KEEPALIVED_HEALTH_SERVICE_NAME }} - } - - #unicast - unicast_peer { - {{ KEEPALIVED_UNICAST_PEER }} - } - #unicast - - virtual_ipaddress { - {{ KEEPALIVED_VIRTUAL_IP }} label {{ KEEPALIVED_INTERFACE }}:VIP - } -} \ No newline at end of file