Merge pull request #258 from galal-hussein/add_etcd_delete_from_k8s

Remove deleted etcd member from k8s and update kube-api with new list of members

cluster/reconcile.go

@@ -24,15 +24,15 @@ func ReconcileCluster(ctx context.Context, kubeCluster, currentCluster *Cluster)
 		return nil
 	}
 
-	if err := reconcileEtcd(ctx, currentCluster, kubeCluster); err != nil {
-		return fmt.Errorf("Failed to reconcile etcd plane: %v", err)
-	}
-
 	kubeClient, err := k8s.NewClient(kubeCluster.LocalKubeConfigPath)
 	if err != nil {
 		return fmt.Errorf("Failed to initialize new kubernetes client: %v", err)
 	}
 
+	if err := reconcileEtcd(ctx, currentCluster, kubeCluster, kubeClient); err != nil {
+		return fmt.Errorf("Failed to reconcile etcd plane: %v", err)
+	}
+
 	if err := reconcileWorker(ctx, currentCluster, kubeCluster, kubeClient); err != nil {
 		return err
 	}
@@ -154,14 +154,18 @@ func reconcileHost(ctx context.Context, toDeleteHost *hosts.Host, worker, etcd b
 	return nil
 }
 
-func reconcileEtcd(ctx context.Context, currentCluster, kubeCluster *Cluster) error {
+func reconcileEtcd(ctx context.Context, currentCluster, kubeCluster *Cluster, kubeClient *kubernetes.Clientset) error {
 	log.Infof(ctx, "[reconcile] Check etcd hosts to be deleted")
 	etcdToDelete := hosts.GetToDeleteHosts(currentCluster.EtcdHosts, kubeCluster.EtcdHosts)
 	for _, etcdHost := range etcdToDelete {
 		if err := services.RemoveEtcdMember(ctx, etcdHost, kubeCluster.EtcdHosts, currentCluster.LocalConnDialerFactory); err != nil {
 			log.Warnf(ctx, "[reconcile] %v", err)
 			continue
 		}
+		if err := hosts.DeleteNode(ctx, etcdHost, kubeClient, etcdHost.IsControl); err != nil {
+			log.Warnf(ctx, "Failed to delete etcd node %s from cluster", etcdHost.Address)
+			continue
+		}
 		// attempting to clean services/files on the host
 		if err := reconcileHost(ctx, etcdHost, false, true, currentCluster.SystemImages[AplineImage], currentCluster.DockerDialerFactory); err != nil {
 			log.Warnf(ctx, "[reconcile] Couldn't clean up etcd node [%s]: %v", etcdHost.Address, err)

services/kubeapi.go

@@ -41,12 +41,13 @@ func buildKubeAPIConfig(host *hosts.Host, kubeAPIService v3.KubeAPIService, etcd
 			"--runtime-config=batch/v2alpha1",
 			"--runtime-config=authentication.k8s.io/v1beta1=true",
 			"--storage-backend=etcd3",
-			"--etcd-servers=" + etcdConnString,
 			"--client-ca-file=" + pki.CACertPath,
 			"--tls-cert-file=" + pki.KubeAPICertPath,
 			"--tls-private-key-file=" + pki.KubeAPIKeyPath,
 			"--service-account-key-file=" + pki.KubeAPIKeyPath},
 	}
+	imageCfg.Cmd = append(imageCfg.Cmd, "--etcd-servers="+etcdConnString)
+
 	if authorizationMode == RBACAuthorizationMode {
 		imageCfg.Cmd = append(imageCfg.Cmd, "--authorization-mode=RBAC")
 	}