feat(cli): implement platform destroy command

cmd/vclusterctl/cmd/platform/destroy.go

@@ -0,0 +1,91 @@
+package platform
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/loft-sh/log"
+	"github.com/loft-sh/vcluster/pkg/cli/flags"
+	"github.com/loft-sh/vcluster/pkg/cli/start"
+	"github.com/loft-sh/vcluster/pkg/platform/clihelper"
+	"github.com/spf13/cobra"
+)
+
+type DestroyCmd struct {
+	start.DeleteOptions
+}
+
+func NewDestroyCmd(globalFlags *flags.GlobalFlags) *cobra.Command {
+	cmd := &DestroyCmd{
+		DeleteOptions: start.DeleteOptions{
+			Options: start.Options{
+				GlobalFlags: globalFlags,
+				Log:         log.GetInstance(),
+				CommandName: "destroy",
+			},
+		},
+	}
+
+	startCmd := &cobra.Command{
+		Use:   "destroy",
+		Short: "Destroy a vCluster platform instance",
+		Long: `########################################################
+############# vcluster platform destroy ##################
+########################################################
+
+Destroys a vCluster platform instance in your Kubernetes cluster.
+
+Please make sure you meet the following requirements
+before running this command:
+
+1. Current kube-context has admin access to the cluster
+2. Helm v3 must be installed
+
+
+VirtualClusterInstances managed with driver helm will be deleted, but the underlying virtual cluster will not be uninstalled
+
+########################################################
+	`,
+		Args: cobra.NoArgs,
+		RunE: func(cobraCmd *cobra.Command, _ []string) error {
+			return cmd.Run(cobraCmd.Context())
+		},
+	}
+
+	startCmd.Flags().StringVar(&cmd.Context, "context", "", "The kube context to use for installation")
+	startCmd.Flags().StringVar(&cmd.Namespace, "namespace", "", "The namespace vCluster platform is installed in")
+	startCmd.Flags().BoolVar(&cmd.DeleteNamespace, "delete-namespace", true, "Whether to delete the namespace or not")
+
+	return startCmd
+}
+
+func (cmd *DestroyCmd) Run(ctx context.Context) error {
+	// initialise clients, verify binaries exist, sanity-check context
+	err := cmd.Options.Prepare()
+	if err != nil {
+		return fmt.Errorf("failed to prepare clients: %w", err)
+	}
+
+	if cmd.Namespace == "" {
+		namespace, err := clihelper.VClusterPlatformInstallationNamespace(ctx)
+		if err != nil {
+			return fmt.Errorf("vCluster platform may not be installed: %w", err)
+		}
+		cmd.Log.Infof("found platform installation in namespace %q", namespace)
+		cmd.Namespace = namespace
+	}
+
+	found, err := clihelper.IsLoftAlreadyInstalled(ctx, cmd.KubeClient, cmd.Namespace)
+	if err != nil {
+		return fmt.Errorf("vCluster platform may not be installed: %w", err)
+	}
+	if !found {
+		return fmt.Errorf("platform not installed in namespace %q", cmd.Namespace)
+	}
+
+	err = start.Destroy(ctx, cmd.DeleteOptions)
+	if err != nil {
+		return fmt.Errorf("failed to destroy platform: %w", err)
+	}
+	return nil
+}

cmd/vclusterctl/cmd/platform/platform.go

@@ -57,10 +57,12 @@ func NewPlatformCmd(globalFlags *flags.GlobalFlags) (*cobra.Command, error) {
 	}
 
 	startCmd := NewStartCmd(globalFlags)
+	destroyCmd := NewDestroyCmd(globalFlags)
 	loginCmd := NewCobraLoginCmd(globalFlags)
 	logoutCmd := NewLogoutCobraCmd(globalFlags)
 
 	platformCmd.AddCommand(startCmd)
+	platformCmd.AddCommand(destroyCmd)
 	platformCmd.AddCommand(NewResetCmd(globalFlags))
 	platformCmd.AddCommand(add.NewAddCmd(globalFlags))
 	platformCmd.AddCommand(NewAccessKeyCmd(globalFlags))

cmd/vclusterctl/cmd/platform/start.go

@@ -21,19 +21,23 @@ import (
 )
 
 type StartCmd struct {
-	start.Options
+	start.StartOptions
 }
 
 func NewStartCmd(globalFlags *flags.GlobalFlags) *cobra.Command {
+	name := "start"
 	cmd := &StartCmd{
-		Options: start.Options{
-			GlobalFlags: globalFlags,
-			Log:         log.GetInstance(),
+		StartOptions: start.StartOptions{
+			Options: start.Options{
+				CommandName: name,
+				GlobalFlags: globalFlags,
+				Log:         log.GetInstance(),
+			},
 		},
 	}
 
 	startCmd := &cobra.Command{
-		Use:   "start",
+		Use:   name,
 		Short: "Start a vCluster platform instance and connect via port-forwarding",
 		Long: `########################################################
 ############# vcluster platform start ##################
@@ -146,7 +150,7 @@ func (cmd *StartCmd) Run(ctx context.Context) error {
 		}
 	}
 
-	return start.NewLoftStarter(cmd.Options).Start(ctx)
+	return start.NewLoftStarter(cmd.StartOptions).Start(ctx)
 }
 
 func (cmd *StartCmd) ensureEmailWithDisclaimer() error {

pkg/cli/start/destroy.go

@@ -0,0 +1,248 @@
+package start
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/loft-sh/log"
+	"github.com/loft-sh/vcluster/pkg/platform/clihelper"
+	apiextensionsv1clientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/dynamic"
+)
+
+// define the order of resource deletion
+var resourceOrder = []string{
+	// instances
+	"virtualclusterinstances",
+	"virtualclusters",
+	"devpodworkspaceinstances",
+	"spaceinstances",
+
+	// templates
+	"virtualclustertemplates",
+	"devpodenvironmenttemplates",
+	"devpodworkspacetemplates",
+	"clusterroletemplates",
+	"spacetemplates",
+	"apps",
+	"spaceconstraints",
+
+	// infra
+	"tasks",
+	"clusterquotas",
+	"projects",
+	"runners",
+	"clusters",
+	"clusteraccesses",
+	"networkpeers",
+
+	// access
+	"teams",
+	"users",
+	"sharedsecrets",
+	"accesskeys",
+	"localclusteraccesses",
+	"localteams",
+	"localusers",
+}
+
+var legacyResourceOrder = []string{
+	"virtualclusters",
+	"spaceconstraints",
+	"localclusteraccesses",
+	"localteams",
+	"localusers",
+}
+
+// DeleteOptions holds cli options for the delete command
+type DeleteOptions struct {
+	Options
+	DeleteNamespace bool
+}
+
+func Destroy(ctx context.Context, opts DeleteOptions) error {
+	// create a dynamic client
+	dynamicClient, err := dynamic.NewForConfig(opts.RestConfig)
+	if err != nil {
+		return err
+	}
+
+	// create a discovery client
+	discoveryClient, err := discovery.NewDiscoveryClientForConfig(opts.RestConfig)
+	if err != nil {
+		return err
+	}
+
+	apiextensionclientset, err := apiextensionsv1clientset.NewForConfig(opts.RestConfig)
+	if err != nil {
+		return err
+	}
+
+	// to compare resources advertised by server vs ones explicitly handled by us
+	clusterResourceSet := sets.New[string]()
+	handledResourceSet := sets.New(resourceOrder...)
+	legacyHandledResourceSet := sets.New(legacyResourceOrder...)
+
+	// get all custom resource definitions in storage.loft.sh
+	resourceList, err := discoveryClient.ServerResourcesForGroupVersion("storage.loft.sh/v1")
+	if err != nil {
+		return err
+	}
+
+	// populate the set
+	for _, resource := range resourceList.APIResources {
+		// don't insert subresources
+		if strings.Contains(resource.Name, "/") {
+			continue
+		}
+		clusterResourceSet.Insert(resource.Name)
+	}
+
+	unhandledResourceSet := clusterResourceSet.Difference(handledResourceSet)
+	if unhandledResourceSet.Len() != 0 {
+		opts.Log.Errorf("some storage.loft.sh resources are unhandled: %v. Try a newer cli version", unhandledResourceSet.UnsortedList())
+		return err
+	}
+
+	for _, resourceName := range resourceOrder {
+		if !clusterResourceSet.Has(resourceName) {
+			// only debug output if legacy resource
+			if legacyHandledResourceSet.Has(resourceName) {
+				opts.Log.Debugf("legacy resource %q not found in discovery, skipping", resourceName)
+			} else {
+				opts.Log.Infof("resource %q not found in discovery, skipping", resourceName)
+			}
+			continue
+		}
+		// list and delete all resources
+		err = deleteAllResourcesAndWait(ctx, dynamicClient, opts.Log, "storage.loft.sh", "v1", resourceName)
+		if err != nil {
+			return err
+		}
+	}
+
+	// helm uninstall and others
+	err = clihelper.UninstallLoft(ctx, opts.KubeClient, opts.RestConfig, opts.Context, opts.Namespace, opts.Log)
+	if err != nil {
+		return err
+	}
+
+	for _, name := range clihelper.DefaultClusterRoles {
+		opts.Log.Infof("deleting clusterrole %q", name)
+		err := opts.KubeClient.RbacV1().ClusterRoles().Delete(ctx, name, metav1.DeleteOptions{})
+		if err != nil && !kerrors.IsNotFound(err) {
+			return fmt.Errorf("failed to delete clusterrole: %w", err)
+		}
+	}
+	for _, name := range clihelper.DefaultClusterRoles {
+		name := name + "-binding"
+		opts.Log.Infof("deleting clusterrolebinding %q", name)
+		err := opts.KubeClient.RbacV1().ClusterRoleBindings().Delete(ctx, name+"-binding", metav1.DeleteOptions{})
+		if err != nil && !kerrors.IsNotFound(err) {
+			return fmt.Errorf("failed to delete clusterrole: %w", err)
+		}
+	}
+
+	err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
+		list, err := apiextensionclientset.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{})
+		if err != nil {
+			return false, err
+		}
+		if len(list.Items) == 0 {
+			return true, nil
+		}
+		for _, object := range list.Items {
+			crdSuffix := ".storage.loft.sh"
+			if !strings.HasSuffix(object.Name, crdSuffix) {
+				continue
+			}
+			expectedResourceName := strings.TrimSuffix(object.Name, crdSuffix)
+			if !handledResourceSet.Has(expectedResourceName) {
+				opts.Log.Errorf("unhandled CRD: %q", object.Name)
+				continue
+			}
+			if !object.GetDeletionTimestamp().IsZero() {
+				opts.Log.Infof("deleted CRD still found: %q", object.GetName())
+				continue
+			}
+			opts.Log.Infof("deleting customresourcedefinition %v", object.GetName())
+			err := apiextensionclientset.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, object.Name, metav1.DeleteOptions{})
+			if err != nil {
+				return false, err
+			}
+		}
+		return false, nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to delete CRDs: %w", err)
+	}
+
+	if opts.DeleteNamespace {
+		opts.Log.Infof("deleting namespace %q", opts.Namespace)
+		err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
+			ns, err := opts.KubeClient.CoreV1().Namespaces().Get(ctx, opts.Namespace, metav1.GetOptions{})
+			if kerrors.IsNotFound(err) {
+				return true, nil
+			} else if err != nil {
+				return false, err
+			}
+
+			if ns.GetDeletionTimestamp().IsZero() {
+				err = opts.KubeClient.CoreV1().Namespaces().Delete(ctx, opts.Namespace, metav1.DeleteOptions{})
+				if err != nil {
+					return false, err
+				}
+			}
+			return false, nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func deleteAllResourcesAndWait(ctx context.Context, dynamicClient dynamic.Interface, log log.Logger, group, version, resource string) error {
+	gvr := schema.GroupVersionResource{Group: group, Version: version, Resource: resource}
+	err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
+		log.Debugf("checking all %q", resource)
+
+		resourceClient := dynamicClient.Resource(gvr)
+		list, err := resourceClient.List(ctx, metav1.ListOptions{})
+		if err != nil {
+			return false, err
+		}
+		if len(list.Items) == 0 {
+			return true, nil
+		}
+		for _, object := range list.Items {
+			if !object.GetDeletionTimestamp().IsZero() {
+				return false, nil
+			}
+			if object.GetNamespace() == "" {
+				log.Infof("deleting %v %v", resource, object.GetName())
+			} else {
+				log.Infof("deleting %v %v/%v", resource, object.GetNamespace(), object.GetName())
+			}
+			err := resourceClient.Namespace(object.GetNamespace()).Delete(ctx, object.GetName(), metav1.DeleteOptions{})
+			if err != nil && !kerrors.IsNotFound(err) {
+				return false, err
+			}
+		}
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}