diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index af50df1e..82c2e4be 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -11,4 +11,7 @@ const ( ChartBasePath = "/var/run/karmada-operator" // KubeconfigBasePath = "/var/run/karmada-operator/kubeconfig". KubeconfigBasePath = ChartBasePath + "/config" + + // RandomNamespace the name of namespace is a random value + RandomNamespace = "install.karmada.io/random-namespace" ) diff --git a/pkg/controller/installer_controller.go b/pkg/controller/installer_controller.go index 4b2dbd03..97078d1d 100644 --- a/pkg/controller/installer_controller.go +++ b/pkg/controller/installer_controller.go @@ -18,6 +18,7 @@ package controller import ( "context" + "reflect" "sync" "time" @@ -34,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" installv1alpha1 "github.com/daocloud/karmada-operator/pkg/apis/install/v1alpha1" + "github.com/daocloud/karmada-operator/pkg/constants" "github.com/daocloud/karmada-operator/pkg/generated/clientset/versioned" installinformers "github.com/daocloud/karmada-operator/pkg/generated/informers/externalversions/install/v1alpha1" installliter "github.com/daocloud/karmada-operator/pkg/generated/listers/install/v1alpha1" @@ -59,7 +61,7 @@ type Controller struct { factory *factory.InstallerFactory - installStore installliter.KarmadaDeploymentLister + installLister installliter.KarmadaDeploymentLister // instalStoreSynced returns true if the kmd store has been synced at least once. // Added as a member to the struct to allow injection for testing. instalStoreSynced cache.InformerSynced @@ -73,7 +75,7 @@ func NewController(kmdClient versioned.Interface, controller := &Controller{ kmdClient: kmdClient, clientset: client, - installStore: karmadaDeploymentInformer.Lister(), + installLister: karmadaDeploymentInformer.Lister(), instalStoreSynced: karmadaDeploymentInformer.Informer().HasSynced, queue: workqueue.NewRateLimitingQueue( workqueue.NewItemExponentialFailureRateLimiter(2*time.Second, 5*time.Second), @@ -188,7 +190,7 @@ func (c *Controller) syncHandler(key string) (err error) { return err } - kmd, err := c.installStore.Get(name) + kmd, err := c.installLister.Get(name) if err != nil { if apierrors.IsNotFound(err) { klog.V(4).Infof("kmd has been deleted: %v", key) @@ -208,10 +210,12 @@ func (c *Controller) syncHandler(key string) (err error) { return err } } - controllerutil.ContainsFinalizer(kmd.DeepCopy(), ControllerFinalizer) - _ = controllerutil.RemoveFinalizer(kmd, ControllerFinalizer) - if _, err := c.kmdClient.InstallV1alpha1().KarmadaDeployments().Update(context.TODO(), kmd, metav1.UpdateOptions{}); err != nil { - return err + + if controllerutil.ContainsFinalizer(kmd.DeepCopy(), ControllerFinalizer) { + _ = controllerutil.RemoveFinalizer(kmd, ControllerFinalizer) + if _, err := c.kmdClient.InstallV1alpha1().KarmadaDeployments().Update(context.TODO(), kmd, metav1.UpdateOptions{}); err != nil { + return err + } } return nil } @@ -221,19 +225,20 @@ func (c *Controller) syncHandler(key string) (err error) { // initDefaultValues init karmadaDeployment default value func (c *Controller) initDefaultValues(kmd *installv1alpha1.KarmadaDeployment) error { var err error - isUpdate := false + kmdClone := kmd.DeepCopy() // add default label karmadadeployments.install.karmada.io/disable-cascading-deletion:true - if kmd.GetLabels() == nil { - kmd.SetLabels(make(map[string]string)) - } - kmdLabels := kmd.GetLabels() - if _, isExist := kmdLabels[DisableCascadingDeletionLabel]; !isExist { - kmdLabels[DisableCascadingDeletionLabel] = "false" - kmd.SetLabels(kmdLabels) - isUpdate = true + + if len(kmd.GetLabels()[DisableCascadingDeletionLabel]) == 0 { + if kmd.GetLabels() == nil { + kmd.Labels = make(map[string]string, 1) + } + kmd.Labels[DisableCascadingDeletionLabel] = "false" } if len(kmd.Spec.ControlPlane.Namespace) == 0 { - kmd.Spec.ControlPlane.Namespace = kmd.Name + "-" + rand.String(5) + if kmd.GetAnnotations() == nil { + kmd.Annotations = make(map[string]string, 1) + } + kmd.Annotations[constants.RandomNamespace] = kmd.Name + "-" + rand.String(5) } if len(kmd.Spec.ControlPlane.ServiceType) == 0 { kmd.Spec.ControlPlane.ServiceType = corev1.ServiceTypeNodePort @@ -242,13 +247,10 @@ func (c *Controller) initDefaultValues(kmd *installv1alpha1.KarmadaDeployment) e // ensure finalizer if !controllerutil.ContainsFinalizer(kmd, ControllerFinalizer) && kmd.DeletionTimestamp.IsZero() { _ = controllerutil.AddFinalizer(kmd, ControllerFinalizer) - isUpdate = true } - if isUpdate { + if !reflect.DeepEqual(kmdClone, kmd) { kmd, err = c.kmdClient.InstallV1alpha1().KarmadaDeployments().Update(context.TODO(), kmd, metav1.UpdateOptions{}) - if err != nil { - return err - } + return err } return nil } diff --git a/pkg/controller/summary/summary_controller.go b/pkg/controller/summary/summary_controller.go index 4a60cb24..058e8933 100644 --- a/pkg/controller/summary/summary_controller.go +++ b/pkg/controller/summary/summary_controller.go @@ -55,8 +55,8 @@ type SummaryController struct { client clientset.Interface kmdClient versioned.Interface - queue workqueue.RateLimitingInterface - installStore installliter.KarmadaDeploymentLister + queue workqueue.RateLimitingInterface + installLister installliter.KarmadaDeploymentLister // instalStoreSynced returns true if the kmd store has been synced at least once. // Added as a member to the struct to allow injection for testing. instalStoreSynced cache.InformerSynced @@ -83,7 +83,7 @@ func NewController(client clientset.Interface, kmdClient versioned.Interface, km DeleteFunc: controller.DeleteEvent, }, ) - controller.installStore = kmdInformer.Lister() + controller.installLister = kmdInformer.Lister() controller.instalStoreSynced = kmdInformer.Informer().HasSynced controller.syncHandler = controller.syncKmdStatusResource @@ -201,7 +201,7 @@ func (rc *SummaryController) syncKmdStatusResource(key string) (bool, error) { return false, err } - kmd, err := rc.installStore.Get(name) + kmd, err := rc.installLister.Get(name) if err != nil { if apierrors.IsNotFound(err) { klog.V(4).Infof("%v has been deleted", key) @@ -282,7 +282,7 @@ func (rc *SummaryController) GetSummaryManager(kmd *installv1alpha1.KarmadaDeplo return nil, err } - manager = NewSummaryManager(kmd.Name, rc.kmdClient, rc.installStore, dynamicClient) + manager = NewSummaryManager(kmd.Name, rc.kmdClient, rc.installLister, dynamicClient) rc.managers[kmd.Name] = manager return manager, nil diff --git a/pkg/controller/summary/summary_manager.go b/pkg/controller/summary/summary_manager.go index 50aeaefd..4c569659 100644 --- a/pkg/controller/summary/summary_manager.go +++ b/pkg/controller/summary/summary_manager.go @@ -63,7 +63,7 @@ type SummaryManager struct { kmdName string kmdClient versioned.Interface - installStore installliter.KarmadaDeploymentLister + installLister installliter.KarmadaDeploymentLister informerManager informermanager.InformerManager closeOnce sync.Once @@ -78,12 +78,12 @@ func (h *NothingResourceEventHandler) OnDelete(obj interface{}) {} func (h *NothingResourceEventHandler) OnUpdate(oldObj, newObj interface{}) {} func NewSummaryManager(kmdName string, kmdClient versioned.Interface, - installStore installliter.KarmadaDeploymentLister, client dynamic.Interface) *SummaryManager { + installLister installliter.KarmadaDeploymentLister, client dynamic.Interface) *SummaryManager { manager := &SummaryManager{ - kmdName: kmdName, - kmdClient: kmdClient, - installStore: installStore, - close: make(chan struct{}), + kmdName: kmdName, + kmdClient: kmdClient, + installLister: installLister, + close: make(chan struct{}), } nothing := &NothingResourceEventHandler{} @@ -147,7 +147,7 @@ func (m *SummaryManager) worker() { } func (m *SummaryManager) processNext() bool { - kmd, err := m.installStore.Get(m.kmdName) + kmd, err := m.installLister.Get(m.kmdName) if err != nil { if apierrors.IsNotFound(err) { klog.V(4).Infof("%v has been deleted", m.kmdName) @@ -408,8 +408,8 @@ func aggregateClusterSummary(clusterSummary *installv1alpha1.ClusterSummary) *in } func checkClusterReady(conditions []metav1.Condition) bool { - for _, condtion := range conditions { - if condtion.Type == clusterv1alpha1.ClusterConditionReady && condtion.Status == metav1.ConditionTrue { + for _, condition := range conditions { + if condition.Type == clusterv1alpha1.ClusterConditionReady && condition.Status == metav1.ConditionTrue { return true } } @@ -418,8 +418,8 @@ func checkClusterReady(conditions []metav1.Condition) bool { } func checkDeploymentReady(conditions []appsv1.DeploymentCondition) bool { - for _, condtion := range conditions { - if condtion.Type == appsv1.DeploymentAvailable && condtion.Status == corev1.ConditionTrue { + for _, condition := range conditions { + if condition.Type == appsv1.DeploymentAvailable && condition.Status == corev1.ConditionTrue { return true } } diff --git a/pkg/installer/helm/install.go b/pkg/installer/helm/install.go index 424fc816..af1d1f47 100644 --- a/pkg/installer/helm/install.go +++ b/pkg/installer/helm/install.go @@ -121,8 +121,7 @@ func (install *installWorkflow) Preflight(kmd *installv1alpha1.KarmadaDeployment if err != nil { klog.ErrorS(err, "[helm-installer]:failed to fetch karmada chart pkg") kmd = installv1alpha1.KarmadaDeploymentNotReady(kmd, installv1alpha1.HelmChartFailedReason, err.Error()) - status.SetStatus(install.kmdClient, kmd) - return err + return status.SetStatus(install.kmdClient, kmd) } if install.values == nil { @@ -132,7 +131,7 @@ func (install *installWorkflow) Preflight(kmd *installv1alpha1.KarmadaDeployment klog.ErrorS(err, "[helm-installer]:failed get ips of kubenetes master node") } - SetChartDefaultValues(values, kmd.Spec.ControlPlane.Namespace, IPs) + SetChartDefaultValues(values, utils.GetControlPlaneNamespace(kmd), IPs) install.values = values } return nil @@ -187,16 +186,16 @@ func (install *installWorkflow) Deploy(kmd *installv1alpha1.KarmadaDeployment) e if err := status.SetStatusPhase(install.kmdClient, kmd, installv1alpha1.DeployingPhase); err != nil { return err } - - ns, err := install.destClient.CoreV1().Namespaces().Get(context.TODO(), kmd.Spec.ControlPlane.Namespace, metav1.GetOptions{}) + controlPlaneNamespace := utils.GetControlPlaneNamespace(kmd) + ns, err := install.destClient.CoreV1().Namespaces().Get(context.TODO(), controlPlaneNamespace, metav1.GetOptions{}) if err != nil { if !apierrors.IsNotFound(err) { - klog.ErrorS(err, "failed to get namespace from dest cluster", "namespace", kmd.Spec.ControlPlane.Namespace) + klog.ErrorS(err, "failed to get namespace from dest cluster", "namespace", controlPlaneNamespace) return err } if ns, err = install.destClient.CoreV1().Namespaces().Create(context.TODO(), - &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: kmd.Spec.ControlPlane.Namespace}}, metav1.CreateOptions{}); err != nil { + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: controlPlaneNamespace}}, metav1.CreateOptions{}); err != nil { return err } } @@ -207,8 +206,7 @@ func (install *installWorkflow) Deploy(kmd *installv1alpha1.KarmadaDeployment) e if err != nil { klog.ErrorS(err, "[helm-installer]:failed to compose chart values") kmd = installv1alpha1.KarmadaDeploymentNotReady(kmd, installv1alpha1.HelmInitFailedReason, err.Error()) - status.SetStatus(install.kmdClient, kmd) - return err + return status.SetStatus(install.kmdClient, kmd) } releaseName := fmt.Sprintf("karmada-%s", kmd.Name) install.release, err = install.helmClient.UpgradeFromPath(install.chartPath, @@ -226,8 +224,7 @@ func (install *installWorkflow) Deploy(kmd *installv1alpha1.KarmadaDeployment) e if err != nil && !strings.Contains(err.Error(), ReleaseExistErrMsg) { klog.ErrorS(err, "[helm-installer]:failed to install karmada chart", "kmd", kmd.Name) kmd = installv1alpha1.KarmadaDeploymentNotReady(kmd, installv1alpha1.HelmReleaseFailedReason, err.Error()) - status.SetStatus(install.kmdClient, kmd) - return err + return status.SetStatus(install.kmdClient, kmd) } // TODO: if community supports installing components together, it will not need install again. @@ -255,8 +252,7 @@ func (install *installWorkflow) Deploy(kmd *installv1alpha1.KarmadaDeployment) e if err != nil { klog.ErrorS(err, "[helm-installer]:failed to install karmada component", "kmd", kmd.Name) kmd = installv1alpha1.KarmadaDeploymentNotReady(kmd, installv1alpha1.HelmReleaseFailedReason, err.Error()) - status.SetStatus(install.kmdClient, kmd) - return err + return status.SetStatus(install.kmdClient, kmd) } } @@ -268,8 +264,7 @@ func (install *installWorkflow) Wait(kmd *installv1alpha1.KarmadaDeployment) err release, err := install.GetRelease(kmd) if err != nil { kmd = installv1alpha1.KarmadaDeploymentNotReady(kmd, installv1alpha1.HelmReleaseFailedReason, err.Error()) - status.SetStatus(install.kmdClient, kmd) - return err + return status.SetStatus(install.kmdClient, kmd) } kmd.Status.Phase = installv1alpha1.WaitingPhase @@ -282,8 +277,7 @@ func (install *installWorkflow) Wait(kmd *installv1alpha1.KarmadaDeployment) err if _, err := waitForPodReady(install.client, release.Namespace, m); err != nil { klog.ErrorS(err, "[helm-installer]:failed to wait ready", "component", m, "namespace", release.Namespace) kmd = installv1alpha1.KarmadaDeploymentNotReady(kmd, installv1alpha1.ReconciliationFailedReason, err.Error()) - status.SetStatus(install.kmdClient, kmd) - return err + return status.SetStatus(install.kmdClient, kmd) } } return nil diff --git a/pkg/installer/helm/uninstall.go b/pkg/installer/helm/uninstall.go index 9ee6cfe8..4ee2ec10 100644 --- a/pkg/installer/helm/uninstall.go +++ b/pkg/installer/helm/uninstall.go @@ -27,7 +27,9 @@ import ( "k8s.io/klog/v2" installv1alpha1 "github.com/daocloud/karmada-operator/pkg/apis/install/v1alpha1" + "github.com/daocloud/karmada-operator/pkg/constants" "github.com/daocloud/karmada-operator/pkg/helm" + "github.com/daocloud/karmada-operator/pkg/utils" ) const ( @@ -116,23 +118,34 @@ func (un *uninstallWorkflow) UninstallComponent(kmd *installv1alpha1.KarmadaDepl // 2. clusterRole/karmada-pre-job // 3. clusterRoleBinding/karmada-pre-job // 4. ns/karmada-system -func (un *uninstallWorkflow) cleanup(kd *installv1alpha1.KarmadaDeployment, release, namespace string) error { - _ = un.destClient.CoreV1().ServiceAccounts(kd.Spec.ControlPlane.Namespace).Delete( +func (un *uninstallWorkflow) cleanup(kmd *installv1alpha1.KarmadaDeployment, release, namespace string) error { + controlPlaneNamespace := utils.GetControlPlaneNamespace(kmd) + err := un.destClient.CoreV1().ServiceAccounts(controlPlaneNamespace).Delete( context.TODO(), fmt.Sprintf("%s-pre-job", release), metav1.DeleteOptions{}) - - _ = un.destClient.RbacV1().ClusterRoles().Delete( + if err != nil { + klog.ErrorS(err, "Failed to delete ServiceAccount(%s/%s)", controlPlaneNamespace, fmt.Sprintf("%s-pre-job", release)) + } + err = un.destClient.RbacV1().ClusterRoles().Delete( context.TODO(), fmt.Sprintf("%s-pre-job", release), metav1.DeleteOptions{}) - - _ = un.destClient.RbacV1().ClusterRoleBindings().Delete( + if err != nil { + klog.ErrorS(err, "Failed to delete sa") + } + err = un.destClient.RbacV1().ClusterRoleBindings().Delete( context.TODO(), fmt.Sprintf("%s-pre-job", release), metav1.DeleteOptions{}) - - _ = un.destClient.CoreV1().Namespaces().Delete( - context.TODO(), namespace, metav1.DeleteOptions{}) - + if err != nil { + klog.ErrorS(err, "Failed to delete sa") + } + if kmd.GetAnnotations()[constants.RandomNamespace] != "" && kmd.Spec.ControlPlane.Namespace == "" { + // Delete only randomly generated namespaces + if err = un.destClient.CoreV1().Namespaces().Delete(context.TODO(), kmd.GetAnnotations()[constants.RandomNamespace], metav1.DeleteOptions{}); err != nil { + klog.ErrorS(err, "Failed delete randomly generated namespaces") + return err + } + } // delete the secret of karmada instance kubeconfig on the cluster. - secretRef := kd.Status.SecretRef + secretRef := kmd.Status.SecretRef if secretRef != nil { - un.client.CoreV1().Secrets(secretRef.Namespace).Delete(context.TODO(), secretRef.Name, metav1.DeleteOptions{}) + return un.client.CoreV1().Secrets(secretRef.Namespace).Delete(context.TODO(), secretRef.Name, metav1.DeleteOptions{}) } // TODO: delete kubeconfig from the directory diff --git a/pkg/installer/helm/values.go b/pkg/installer/helm/values.go index 355bf5ff..3b769fb2 100644 --- a/pkg/installer/helm/values.go +++ b/pkg/installer/helm/values.go @@ -246,7 +246,7 @@ func Convert_KarmadaDeployment_To_Values(kmd *installv1alpha1.KarmadaDeployment) return values } -// setDefaultValues set apiserver default values +// SetChartDefaultValues set apiserver default values // set default external ip // the certificate expires in 10 years func SetChartDefaultValues(v *Values, releaseNamespace string, externalHosts []string) { diff --git a/pkg/utils/namespace.go b/pkg/utils/namespace.go new file mode 100644 index 00000000..08420e99 --- /dev/null +++ b/pkg/utils/namespace.go @@ -0,0 +1,16 @@ +package utils + +import ( + installv1alpha1 "github.com/daocloud/karmada-operator/pkg/apis/install/v1alpha1" + "github.com/daocloud/karmada-operator/pkg/constants" +) + +func GetControlPlaneNamespace(kmd *installv1alpha1.KarmadaDeployment) string { + if kmd.Spec.ControlPlane.Namespace != "" { + return kmd.Spec.ControlPlane.Namespace + } + if ns, exist := kmd.Annotations[constants.RandomNamespace]; exist { + return ns + } + return "default" +}