diff --git a/cmd/ciReleaseCleanfailed.go b/cmd/ciReleaseCleanfailed.go index a22a9c8..3f6924a 100644 --- a/cmd/ciReleaseCleanfailed.go +++ b/cmd/ciReleaseCleanfailed.go @@ -1,9 +1,17 @@ package cmd import ( + "context" "fmt" + "log" + "os" + helmclient "github.com/mittwald/go-helm-client" "github.com/spf13/cobra" + "github.com/wunderio/silta-cli/internal/common" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) var ciReleaseCleanfailedCmd = &cobra.Command{ @@ -13,40 +21,75 @@ var ciReleaseCleanfailedCmd = &cobra.Command{ releaseName, _ := cmd.Flags().GetString("release-name") namespace, _ := cmd.Flags().GetString("namespace") - command := fmt.Sprintf(` - NAMESPACE='%s' - RELEASE_NAME='%s' - - failed_revision=$(helm list -n "${NAMESPACE}" --failed --pending --filter="(\s|^)(${RELEASE_NAME})(\s|$)" | tail -1 | cut -f3) - - if [[ "${failed_revision}" -eq 1 ]]; then - # Remove any existing post-release hook, since it's technically not part of the release. - kubectl delete job -n "${NAMESPACE}" "${RELEASE_NAME}-post-release" 2> /dev/null || true - - echo "Removing failed first release." - helm delete -n "${NAMESPACE}" "${RELEASE_NAME}" - - echo "Delete persistent volume claims left over from statefulsets." - kubectl delete pvc -n "${NAMESPACE}" -l release="${RELEASE_NAME}" - kubectl delete pvc -n "${NAMESPACE}" -l app="${RELEASE_NAME}-es" - - echo -n "Waiting for volumes to be deleted." - until [[ -z $(kubectl get pv | grep "${NAMESPACE}/${RELEASE_NAME}-") ]] - do - echo -n "." - sleep 5 - done - fi - - # Workaround for previous Helm release stuck in pending state - pending_release=$(helm list -n "${NAMESPACE}" --pending --filter="(\s|^)(${RELEASE_NAME})(\s|$)"| tail -1 | cut -f1) - - if [[ "${pending_release}" == "${RELEASE_NAME}" ]]; then - secret_to_delete=$(kubectl get secret -l owner=helm,status=pending-upgrade,name="${RELEASE_NAME}" -n "${NAMESPACE}" | awk '{print $1}' | grep -v NAME) - kubectl delete secret -n "${NAMESPACE}" "${secret_to_delete}" - fi - `, namespace, releaseName) - pipedExec(command, "", "ERROR: ", debug) + // ---- + + homeDir, err := os.UserHomeDir() + if err != nil { + log.Fatalf("cannot read user home dir") + } + kubeConfigPath := homeDir + "/.kube/config" + + kubeConfig, err := os.ReadFile(kubeConfigPath) + if err != nil { + log.Fatalf("cannot read kubeConfig from path") + } + + //k8s go client init logic + config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) + if err != nil { + log.Fatalf("cannot read kubeConfig from path: %s", err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Fatalf("cannot initialize k8s client: %s", err) + } + + //Helm client init logic + opt := &helmclient.KubeConfClientOptions{ + Options: &helmclient.Options{ + Namespace: namespace, + RepositoryCache: "/tmp/.helmcache", + RepositoryConfig: "/tmp/.helmrepo", + Debug: false, + Linting: false, // Change this to false if you don't want linting. + }, + KubeContext: "", + KubeConfig: kubeConfig, + } + + helmClient, err := helmclient.NewClientFromKubeConf(opt) + if err != nil { + log.Fatalf("Cannot create client from kubeConfig") + } + + // Get release info + release, err := helmClient.GetRelease(releaseName) + if err != nil { + return // Release not found or there was an error + } + + // Check if there's only one revision and it's failed + if release.Version == 1 && release.Info.Status == "failed" { + + fmt.Println("Removing failed first release.") + + // Remove release + common.UninstallHelmRelease(clientset, helmClient, releaseName, namespace, true) + } + + // Workaround for previous Helm release stuck in pending state + // This is a workaround for a known issue with Helm where a release can get stuck in a pending-upgrade state + // and the secret is not deleted. This is a workaround to delete the secret if it exists. + if release.Info.Status == "pending-upgrade" { + secretName := fmt.Sprintf("%s.%s.v%d", releaseName, namespace, release.Version) + if err == nil { + fmt.Printf("Deleting secret %s\n", secretName) + err := clientset.CoreV1().Secrets(namespace).Delete(context.TODO(), secretName, v1.DeleteOptions{}) + if err != nil { + log.Fatalf("Error deleting secret %s: %s", secretName, err) + } + } + } }, } diff --git a/cmd/ciReleaseDelete.go b/cmd/ciReleaseDelete.go index 3824668..39e80e6 100644 --- a/cmd/ciReleaseDelete.go +++ b/cmd/ciReleaseDelete.go @@ -1,13 +1,12 @@ package cmd import ( - "context" "log" "os" helmclient "github.com/mittwald/go-helm-client" "github.com/spf13/cobra" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/wunderio/silta-cli/internal/common" "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // gcp auth provider "k8s.io/client-go/tools/clientcmd" @@ -60,61 +59,9 @@ var ciReleaseDeleteCmd = &cobra.Command{ log.Fatalf("Cannot create client from kubeConfig") } - //Uninstall Helm release - uninstallErr := helmClient.UninstallReleaseByName(releaseName) - if uninstallErr != nil { - log.Fatalf("Error removing a release:%s", uninstallErr) - } - - //Delete pre-release jobs - selectorLabels := []string{ - "release", - "app.kubernetes.io/instance", - } - - for _, l := range selectorLabels { - selector := l + "=" + releaseName - list, err := clientset.BatchV1().Jobs(namespace).List(context.TODO(), v1.ListOptions{ - LabelSelector: selector, - }) - if err != nil { - log.Fatalf("Error getting the list of jobs: %s", err) - } - for _, v := range list.Items { - log.Printf("Deleting job: %s", v.Name) - propagationPolicy := v1.DeletePropagationBackground - clientset.BatchV1().Jobs(namespace).Delete(context.TODO(), v.Name, v1.DeleteOptions{PropagationPolicy: &propagationPolicy}) - } - } - - //Delete PVCs - if deletePVCs { - - PVC_client := clientset.CoreV1().PersistentVolumeClaims(namespace) - - selectorLabels := []string{ - "app", - "release", - "app.kubernetes.io/instance", - } - - for _, l := range selectorLabels { - selector := l + "=" + releaseName - if l == "app" { - selector = l + "=" + releaseName + "-es" - } - list, err := PVC_client.List(context.TODO(), v1.ListOptions{ - LabelSelector: selector, - }) - if err != nil { - log.Fatalf("Error getting the list of PVCs: %s", err) - } - - for _, v := range list.Items { - log.Printf("Deleting PVC: %s", v.Name) - PVC_client.Delete(context.TODO(), v.Name, v1.DeleteOptions{}) - } - } + err = common.UninstallHelmRelease(clientset, helmClient, releaseName, namespace, deletePVCs) + if err != nil { + log.Fatalf("Error removing a release: %s", err) } }, diff --git a/cmd/ciReleaseDeploy.go b/cmd/ciReleaseDeploy.go index 1c3689f..af83777 100644 --- a/cmd/ciReleaseDeploy.go +++ b/cmd/ciReleaseDeploy.go @@ -168,6 +168,7 @@ var ciReleaseDeployCmd = &cobra.Command{ } // Create namespace if it doesn't exist + // Describe namespace _, err = clientset.CoreV1().Namespaces().Get(context.TODO(), namespace, v1meta.GetOptions{}) if err != nil { _, err = clientset.CoreV1().Namespaces().Create(context.TODO(), &v1core.Namespace{ @@ -176,7 +177,7 @@ var ciReleaseDeployCmd = &cobra.Command{ }, }, v1meta.CreateOptions{}) if err != nil { - log.Fatalf("cannot create namespace: %s", err) + log.Printf("cannot create namespace: %s\n", err) } } } diff --git a/internal/common/ciReleaseFunctions.go b/internal/common/ciReleaseFunctions.go new file mode 100644 index 0000000..a6c9d6c --- /dev/null +++ b/internal/common/ciReleaseFunctions.go @@ -0,0 +1,70 @@ +package common + +import ( + "context" + "log" + + helmclient "github.com/mittwald/go-helm-client" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// UninstallRelease removes a Helm release and related resources +// Note: namespace is inferred from the helmclient.Options struct but set here for kubernetes clientset actions +func UninstallHelmRelease(clientset *kubernetes.Clientset, helmClient helmclient.Client, releaseName string, namespace string, deletePVCs bool) error { + + // Uninstall helm release. Namespace and other context is provided via the + // helmclient.Options struct when instantiating a client. + // Do not bail when release removal fails, remove related resources anyway. + err := helmClient.UninstallReleaseByName(releaseName) + if err != nil { + log.Printf("Failed to remove helm release: %s", err) + } + + // Delete related jobs + selectorLabels := []string{ + "release", + "app.kubernetes.io/instance", + } + + for _, l := range selectorLabels { + selector := l + "=" + releaseName + list, _ := clientset.BatchV1().Jobs(namespace).List(context.TODO(), v1.ListOptions{ + LabelSelector: selector, + }) + for _, v := range list.Items { + log.Printf("Removing job: %s", v.Name) + propagationPolicy := v1.DeletePropagationBackground + clientset.BatchV1().Jobs(namespace).Delete(context.TODO(), v.Name, v1.DeleteOptions{PropagationPolicy: &propagationPolicy}) + } + } + + if deletePVCs { + + // Find and remove related PVC's by release name label + PVC_client := clientset.CoreV1().PersistentVolumeClaims(namespace) + + selectorLabels = []string{ + "app", + "release", + "app.kubernetes.io/instance", + } + + for _, l := range selectorLabels { + selector := l + "=" + releaseName + if l == "app" { + selector = l + "=" + releaseName + "-es" + } + list, _ := PVC_client.List(context.TODO(), v1.ListOptions{ + LabelSelector: selector, + }) + + for _, v := range list.Items { + log.Printf("Removing PVC: %s", v.Name) + PVC_client.Delete(context.TODO(), v.Name, v1.DeleteOptions{}) + } + } + } + + return nil +}