backend: Fix reloading of clusters in kubeconfig

When a context was being added and removed from KUBECONFIG file, it was still visible in UI. This removes the cluster from store and returns correct list of clusters. Fixes #2541 Signed-off-by: Kautilya Tripathi <[email protected]>
headlamp-k8s · Nov 19, 2024 · c2620c4 · c2620c4
1 parent 8fb46f6
commit c2620c4
Show file tree

Hide file tree

Showing 3 changed files with 144 additions and 48 deletions.
diff --git a/backend/pkg/kubeconfig/file_test.go b/backend/pkg/kubeconfig/file_test.go
@@ -10,6 +10,26 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 )
 
+const clusterConf = `apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: dGVzdA==
+    server: https://kubernetes.docker.internal:6443
+  name: random-cluster-4
+contexts:
+- context:
+    cluster: random-cluster-4
+    user: random-cluster-4
+  name: random-cluster-4
+current-context: random-cluster-4
+kind: Config
+preferences: {}
+users:
+- name: random-cluster-4
+  user:
+  client-certificate-data: dGVzdA==
+  client-key-data: dGVzdA==`
+
 func TestWriteToFile(t *testing.T) {
 	// create kubeconfig3 file that doesn't exist
 	conf, err := clientcmd.Load([]byte(clusterConf))

diff --git a/backend/pkg/kubeconfig/watcher.go b/backend/pkg/kubeconfig/watcher.go
@@ -1,6 +1,7 @@
 package kubeconfig
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"time"
@@ -48,14 +49,13 @@ func LoadAndWatchFiles(kubeConfigStore ContextStore, paths string, source int) {
 		case event := <-watcher.Events:
 			triggers := []fsnotify.Op{fsnotify.Create, fsnotify.Write, fsnotify.Remove, fsnotify.Rename}
 			for _, trigger := range triggers {
-				trigger := trigger
 				if event.Op.Has(trigger) {
 					logger.Log(logger.LevelInfo, map[string]string{"event": event.Name},
 						nil, "watcher: kubeconfig file changed, reloading contexts")
 
-					err := LoadAndStoreKubeConfigs(kubeConfigStore, paths, source)
+					err := syncContexts(kubeConfigStore, paths, source)
 					if err != nil {
-						logger.Log(logger.LevelError, nil, err, "watcher: error loading kubeconfig files")
+						logger.Log(logger.LevelError, nil, err, "watcher: error synchronizing contexts")
 					}
 				}
 			}
@@ -106,3 +106,52 @@ func addFilesToWatcher(watcher *fsnotify.Watcher, paths []string) {
 		}
 	}
 }
+
+// syncContexts synchronizes the contexts in the store with the ones in the kubeconfig files.
+func syncContexts(kubeConfigStore ContextStore, paths string, source int) error {
+	// First read all kubeconfig files to get new contexts
+	newContexts, _, err := LoadContextsFromMultipleFiles(paths, source)
+	if err != nil {
+		return fmt.Errorf("error reading kubeconfig files: %v", err)
+	}
+
+	// Get existing contexts from store
+	existingContexts, err := kubeConfigStore.GetContexts()
+	if err != nil {
+		return fmt.Errorf("error getting existing contexts: %v", err)
+	}
+
+	// Find and remove contexts that no longer exist in the kubeconfig
+	// but only for contexts that came from KubeConfig source
+	for _, existingCtx := range existingContexts {
+		// Skip contexts from other sources
+		if existingCtx.Source != KubeConfig {
+			continue
+		}
+
+		found := false
+
+		for _, newCtx := range newContexts {
+			if existingCtx.Name == newCtx.Name {
+				found = true
+
+				break
+			}
+		}
+
+		if !found {
+			err := kubeConfigStore.RemoveContext(existingCtx.Name)
+			if err != nil {
+				logger.Log(logger.LevelError, nil, err, "error removing context")
+			}
+		}
+	}
+
+	// Now load and store the new configurations
+	err = LoadAndStoreKubeConfigs(kubeConfigStore, paths, source)
+	if err != nil {
+		return fmt.Errorf("error loading kubeconfig files: %v", err)
+	}
+
+	return nil
+}
diff --git a/backend/pkg/kubeconfig/watcher_test.go b/backend/pkg/kubeconfig/watcher_test.go
@@ -1,7 +1,6 @@
 package kubeconfig_test
 
 import (
-	"os"
 	"runtime"
 	"strings"
 	"testing"
@@ -10,30 +9,12 @@ import (
 	"github.com/headlamp-k8s/headlamp/backend/pkg/kubeconfig"
 	"github.com/stretchr/testify/require"
 	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 )
 
-const clusterConf = `apiVersion: v1
-clusters:
-- cluster:
-    certificate-authority-data: dGVzdA==
-    server: https://kubernetes.docker.internal:6443
-  name: random-cluster-4
-contexts:
-- context:
-    cluster: random-cluster-4
-    user: random-cluster-4
-  name: random-cluster-4
-current-context: random-cluster-4
-kind: Config
-preferences: {}
-users:
-- name: random-cluster-4
-  user:
-  client-certificate-data: dGVzdA==
-  client-key-data: dGVzdA==`
-
+//nolint:funlen
 func TestWatchAndLoadFiles(t *testing.T) {
-	paths := []string{"./test_data/kubeconfig1", "./test_data/kubeconfig2", "./test_data/kubeconfig3"}
+	paths := []string{"./test_data/kubeconfig1", "./test_data/kubeconfig2"}
 
 	var path string
 	if runtime.GOOS == "windows" {
@@ -46,37 +27,83 @@ func TestWatchAndLoadFiles(t *testing.T) {
 
 	go kubeconfig.LoadAndWatchFiles(kubeConfigStore, path, kubeconfig.KubeConfig)
 
-	// SLeep so the config file has a different time stamp.
-	time.Sleep(5 * time.Second)
+	// Test adding a context
+	t.Run("Add context", func(t *testing.T) {
+		// Sleep to ensure watcher is ready
+		time.Sleep(2 * time.Second)
+
+		// Read existing config
+		config, err := clientcmd.LoadFromFile("./test_data/kubeconfig1")
+		require.NoError(t, err)
 
-	// create kubeconfig3 file that doesn't exist
-	conf, err := clientcmd.Load([]byte(clusterConf))
-	require.NoError(t, err)
-	require.NotNil(t, conf)
+		// Add new context
+		config.Contexts["random-cluster-4"] = &clientcmdapi.Context{
+			Cluster:  "docker-desktop", // reuse existing cluster
+			AuthInfo: "docker-desktop", // reuse existing auth
+		}
 
-	err = clientcmd.WriteToFile(*conf, "./test_data/kubeconfig3")
-	require.NoError(t, err)
+		// Write back to file
+		err = clientcmd.WriteToFile(*config, "./test_data/kubeconfig1")
+		require.NoError(t, err)
 
-	t.Log("created kubeconfig3 file")
+		// Wait for context to be added
+		found := false
 
-	// check if kubeconfig3 is loaded
-	context, err := kubeConfigStore.GetContext("random-cluster-4")
+		for i := 0; i < 20; i++ {
+			context, err := kubeConfigStore.GetContext("random-cluster-4")
+			if err == nil && context != nil {
+				found = true
+				break
+			}
 
-	// loop for until GetContext returns "random-cluster-4" or 30 seconds has past
-	for i := 0; i < 30; i++ {
-		if err == nil && context.Name == "random-cluster-4" {
-			break
+			time.Sleep(500 * time.Millisecond)
 		}
 
-		time.Sleep(1 * time.Second)
+		require.True(t, found, "Context should have been added")
+	})
 
-		context, err = kubeConfigStore.GetContext("random-cluster-4")
-	}
+	// Test removing a context
+	t.Run("Remove context", func(t *testing.T) {
+		// Verify context exists before removal
+		context, err := kubeConfigStore.GetContext("random-cluster-4")
+		require.NoError(t, err)
+		require.NotNil(t, context)
+
+		// Read existing config
+		config, err := clientcmd.LoadFromFile("./test_data/kubeconfig1")
+		require.NoError(t, err)
+
+		// Remove context
+		delete(config.Contexts, "random-cluster-4")
+
+		// Write back to file
+		err = clientcmd.WriteToFile(*config, "./test_data/kubeconfig1")
+		require.NoError(t, err)
 
-	require.NoError(t, err)
-	require.Equal(t, "random-cluster-4", context.Name)
+		// Wait for context to be removed
+		removed := false
 
-	// delete kubeconfig3 file
-	err = os.Remove("./test_data/kubeconfig3")
-	require.NoError(t, err)
+		for i := 0; i < 20; i++ {
+			_, err = kubeConfigStore.GetContext("random-cluster-4")
+			if err != nil {
+				removed = true
+				break
+			}
+
+			time.Sleep(500 * time.Millisecond)
+		}
+
+		require.True(t, removed, "Context should have been removed")
+	})
+
+	// Cleanup in case test fails
+	defer func() {
+		config, err := clientcmd.LoadFromFile("./test_data/kubeconfig1")
+		if err == nil {
+			delete(config.Contexts, "random-cluster-4")
+
+			err = clientcmd.WriteToFile(*config, "./test_data/kubeconfig1")
+			require.NoError(t, err)
+		}
+	}()
 }