Merge pull request #410 from galal-hussein/secure_kubelet

Secure kubelet port access
rancher · Mar 12, 2018 · bc05bc2 · bc05bc2
2 parents 486cea8 + fcefbf7
commit bc05bc2
Show file tree

Hide file tree

Showing 9 changed files with 40 additions and 14 deletions.
diff --git a/cluster/cluster.go b/cluster/cluster.go
@@ -100,6 +100,7 @@ func (c *Cluster) DeployWorkerPlane(ctx context.Context) error {
 		c.PrivateRegistriesMap,
 		processMap,
 		kubeletProcessHostMap,
+		c.Certificates,
 	); err != nil {
 		return fmt.Errorf("[workerPlane] Failed to bring up Worker Plane: %v", err)
 	}

diff --git a/cluster/plan.go b/cluster/plan.go
@@ -90,6 +90,8 @@ func (c *Cluster) BuildKubeAPIProcess() v3.Process {
 		"--client-ca-file=" + pki.GetCertPath(pki.CACertName),
 		"--tls-cert-file=" + pki.GetCertPath(pki.KubeAPICertName),
 		"--tls-private-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
+		"--kubelet-client-certificate=" + pki.GetCertPath(pki.KubeAPICertName),
+		"--kubelet-client-key=" + pki.GetKeyPath(pki.KubeAPICertName),
 		"--service-account-key-file=" + pki.GetKeyPath(pki.KubeAPICertName),
 	}
 	args := []string{
@@ -201,6 +203,8 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host) v3.Process {
 		"--allow-privileged=true",
 		"--cloud-provider=",
 		"--kubeconfig=" + pki.GetConfigPath(pki.KubeNodeCertName),
+		"--client-ca-file=" + pki.GetCertPath(pki.CACertName),
+		"--anonymous-auth=false",
 		"--volume-plugin-dir=/var/lib/kubelet/volumeplugins",
 		"--require-kubeconfig=True",
 		"--fail-swap-on=" + strconv.FormatBool(c.Services.Kubelet.FailSwapOn),

diff --git a/services/healthcheck.go b/services/healthcheck.go
@@ -12,7 +12,9 @@ import (
 
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/log"
+	"github.com/rancher/rke/pki"
 	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/util/cert"
 )
 
 const (
@@ -22,13 +24,23 @@ const (
 	HTTPSProtoPrefix = "https://"
 )
 
-func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, localConnDialerFactory hosts.DialerFactory, url string) error {
+func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, localConnDialerFactory hosts.DialerFactory, url string, certMap map[string]pki.CertificatePKI) error {
 	log.Infof(ctx, "[healthcheck] Start Healthcheck on service [%s] on host [%s]", serviceName, host.Address)
+	var x509Pair tls.Certificate
+
 	port, err := getPortFromURL(url)
 	if err != nil {
 		return err
 	}
-	client, err := getHealthCheckHTTPClient(host, port, localConnDialerFactory)
+	if serviceName == KubeletContainerName {
+		certificate := cert.EncodeCertPEM(certMap[pki.KubeNodeCertName].Certificate)
+		key := cert.EncodePrivateKeyPEM(certMap[pki.KubeNodeCertName].Key)
+		x509Pair, err = tls.X509KeyPair(certificate, key)
+		if err != nil {
+			return err
+		}
+	}
+	client, err := getHealthCheckHTTPClient(host, port, localConnDialerFactory, &x509Pair)
 	if err != nil {
 		return fmt.Errorf("Failed to initiate new HTTP client for service [%s] for host [%s]", serviceName, host.Address)
 	}
@@ -44,7 +56,7 @@ func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, l
 	return fmt.Errorf("Failed to verify healthcheck: %v", err)
 }
 
-func getHealthCheckHTTPClient(host *hosts.Host, port int, localConnDialerFactory hosts.DialerFactory) (*http.Client, error) {
+func getHealthCheckHTTPClient(host *hosts.Host, port int, localConnDialerFactory hosts.DialerFactory, x509KeyPair *tls.Certificate) (*http.Client, error) {
 	host.LocalConnPort = port
 	var factory hosts.DialerFactory
 	if localConnDialerFactory == nil {
@@ -56,10 +68,17 @@ func getHealthCheckHTTPClient(host *hosts.Host, port int, localConnDialerFactory
 	if err != nil {
 		return nil, fmt.Errorf("Failed to create a dialer for host [%s]: %v", host.Address, err)
 	}
+	tlsConfig := &tls.Config{InsecureSkipVerify: true}
+	if x509KeyPair != nil {
+		tlsConfig = &tls.Config{
+			InsecureSkipVerify: true,
+			Certificates:       []tls.Certificate{*x509KeyPair},
+		}
+	}
 	return &http.Client{
 		Transport: &http.Transport{
 			Dial:            dialer,
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			TLSClientConfig: tlsConfig,
 		},
 	}, nil
 }

diff --git a/services/kubeapi.go b/services/kubeapi.go
@@ -14,7 +14,7 @@ func runKubeAPI(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, p
 	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeAPIContainerName, host.Address, ControlRole, prsMap); err != nil {
 		return err
 	}
-	return runHealthcheck(ctx, host, KubeAPIContainerName, df, healthCheckURL)
+	return runHealthcheck(ctx, host, KubeAPIContainerName, df, healthCheckURL, nil)
 }
 
 func removeKubeAPI(ctx context.Context, host *hosts.Host) error {

diff --git a/services/kubecontroller.go b/services/kubecontroller.go
@@ -13,7 +13,7 @@ func runKubeController(ctx context.Context, host *hosts.Host, df hosts.DialerFac
 	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeControllerContainerName, host.Address, ControlRole, prsMap); err != nil {
 		return err
 	}
-	return runHealthcheck(ctx, host, KubeControllerContainerName, df, healthCheckURL)
+	return runHealthcheck(ctx, host, KubeControllerContainerName, df, healthCheckURL, nil)
 }
 
 func removeKubeController(ctx context.Context, host *hosts.Host) error {

diff --git a/services/kubelet.go b/services/kubelet.go
@@ -5,15 +5,16 @@ import (
 
 	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
+	"github.com/rancher/rke/pki"
 	"github.com/rancher/types/apis/management.cattle.io/v3"
 )
 
-func runKubelet(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, kubeletProcess v3.Process) error {
+func runKubelet(ctx context.Context, host *hosts.Host, df hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, kubeletProcess v3.Process, certMap map[string]pki.CertificatePKI) error {
 	imageCfg, hostCfg, healthCheckURL := GetProcessConfig(kubeletProcess)
 	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeletContainerName, host.Address, WorkerRole, prsMap); err != nil {
 		return err
 	}
-	return runHealthcheck(ctx, host, KubeletContainerName, df, healthCheckURL)
+	return runHealthcheck(ctx, host, KubeletContainerName, df, healthCheckURL, certMap)
 }
 
 func removeKubelet(ctx context.Context, host *hosts.Host) error {

diff --git a/services/kubeproxy.go b/services/kubeproxy.go
@@ -13,7 +13,7 @@ func runKubeproxy(ctx context.Context, host *hosts.Host, df hosts.DialerFactory,
 	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeproxyContainerName, host.Address, WorkerRole, prsMap); err != nil {
 		return err
 	}
-	return runHealthcheck(ctx, host, KubeproxyContainerName, df, healthCheckURL)
+	return runHealthcheck(ctx, host, KubeproxyContainerName, df, healthCheckURL, nil)
 }
 
 func removeKubeproxy(ctx context.Context, host *hosts.Host) error {

diff --git a/services/scheduler.go b/services/scheduler.go
@@ -13,7 +13,7 @@ func runScheduler(ctx context.Context, host *hosts.Host, df hosts.DialerFactory,
 	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, SchedulerContainerName, host.Address, ControlRole, prsMap); err != nil {
 		return err
 	}
-	return runHealthcheck(ctx, host, SchedulerContainerName, df, healthCheckURL)
+	return runHealthcheck(ctx, host, SchedulerContainerName, df, healthCheckURL, nil)
 }
 
 func removeScheduler(ctx context.Context, host *hosts.Host) error {

diff --git a/services/workerplane.go b/services/workerplane.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/log"
+	"github.com/rancher/rke/pki"
 	"github.com/rancher/types/apis/management.cattle.io/v3"
 	"golang.org/x/sync/errgroup"
 )
@@ -13,7 +14,7 @@ const (
 	unschedulableEtcdTaint = "node-role.kubernetes.io/etcd=true:NoExecute"
 )
 
-func RunWorkerPlane(ctx context.Context, allHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, kubeletProcessHostMap map[*hosts.Host]v3.Process) error {
+func RunWorkerPlane(ctx context.Context, allHosts []*hosts.Host, localConnDialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, kubeletProcessHostMap map[*hosts.Host]v3.Process, certMap map[string]pki.CertificatePKI) error {
 	log.Infof(ctx, "[%s] Building up Worker Plane..", WorkerRole)
 	var errgrp errgroup.Group
 	for _, host := range allHosts {
@@ -26,7 +27,7 @@ func RunWorkerPlane(ctx context.Context, allHosts []*hosts.Host, localConnDialer
 		hostProcessMap := copyProcessMap(processMap)
 		errgrp.Go(func() error {
 			hostProcessMap[KubeletContainerName] = kubeletProcessHostMap[runHost]
-			return doDeployWorkerPlane(ctx, runHost, localConnDialerFactory, prsMap, hostProcessMap)
+			return doDeployWorkerPlane(ctx, runHost, localConnDialerFactory, prsMap, hostProcessMap, certMap)
 		})
 	}
 	if err := errgrp.Wait(); err != nil {
@@ -65,7 +66,7 @@ func RemoveWorkerPlane(ctx context.Context, workerHosts []*hosts.Host, force boo
 
 func doDeployWorkerPlane(ctx context.Context, host *hosts.Host,
 	localConnDialerFactory hosts.DialerFactory,
-	prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process) error {
+	prsMap map[string]v3.PrivateRegistry, processMap map[string]v3.Process, certMap map[string]pki.CertificatePKI) error {
 	// run nginx proxy
 	if !host.IsControl {
 		if err := runNginxProxy(ctx, host, prsMap, processMap[NginxProxyContainerName]); err != nil {
@@ -77,7 +78,7 @@ func doDeployWorkerPlane(ctx context.Context, host *hosts.Host,
 		return err
 	}
 	// run kubelet
-	if err := runKubelet(ctx, host, localConnDialerFactory, prsMap, processMap[KubeletContainerName]); err != nil {
+	if err := runKubelet(ctx, host, localConnDialerFactory, prsMap, processMap[KubeletContainerName], certMap); err != nil {
 		return err
 	}
 	return runKubeproxy(ctx, host, localConnDialerFactory, prsMap, processMap[KubeproxyContainerName])