From bff14248c7c67ed344beeb4ea40b36fa03aace99 Mon Sep 17 00:00:00 2001 From: doodgeMatvey Date: Mon, 5 Feb 2024 15:46:15 +0200 Subject: [PATCH] issue-638, check for operator version and basic image update were implemented --- .secrets.baseline | 32 ++--- apis/clusters/v1beta1/postgresql_types.go | 23 ---- config/manager/manager.yaml | 1 + config/rbac/role.yaml | 2 + controllers/clusters/cadence_controller.go | 1 + main.go | 13 ++ pkg/models/operator.go | 3 + pkg/models/upgrade_check.go | 11 ++ pkg/scheduler/scheduler.go | 2 + pkg/upgradecheck/upgrade_check.go | 146 +++++++++++++++++++++ 10 files changed, 195 insertions(+), 39 deletions(-) create mode 100644 pkg/models/upgrade_check.go create mode 100644 pkg/upgradecheck/upgrade_check.go diff --git a/.secrets.baseline b/.secrets.baseline index e2dcb91c9..5a819ca17 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -324,21 +324,21 @@ "filename": "apis/clusters/v1beta1/postgresql_types.go", "hashed_secret": "5ffe533b830f08a0326348a9160afafc8ada44db", "is_verified": false, - "line_number": 375 + "line_number": 352 }, { "type": "Secret Keyword", "filename": "apis/clusters/v1beta1/postgresql_types.go", "hashed_secret": "a3d7d4a96d18c8fc5a1cf9c9c01c45b4690b4008", "is_verified": false, - "line_number": 381 + "line_number": 358 }, { "type": "Secret Keyword", "filename": "apis/clusters/v1beta1/postgresql_types.go", "hashed_secret": "a57ce131bd944bdf8ba2f2f93e179dc416ed0315", "is_verified": false, - "line_number": 501 + "line_number": 478 } ], "apis/clusters/v1beta1/redis_types.go": [ @@ -395,7 +395,7 @@ "filename": "apis/clusters/v1beta1/zz_generated.deepcopy.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 2101 + "line_number": 2174 } ], "apis/kafkamanagement/v1beta1/kafkauser_types.go": [ @@ -536,14 +536,14 @@ "filename": "controllers/clusters/cadence_controller.go", "hashed_secret": "bcf196cdeea4d7ed8b04dcbbd40111eb5e9abeac", "is_verified": false, - "line_number": 773 + "line_number": 774 }, { "type": "Secret Keyword", "filename": "controllers/clusters/cadence_controller.go", "hashed_secret": "192d703e91a60432ce06bfe26adfd12f5c7b931f", "is_verified": false, - "line_number": 815 + "line_number": 816 } ], "controllers/clusters/datatest/kafka_v1beta1.yaml": [ @@ -1066,63 +1066,63 @@ "filename": "pkg/models/operator.go", "hashed_secret": "b021a4982481503b77dfa4c9e34dbd935c5121cc", "is_verified": false, - "line_number": 32 + "line_number": 35 }, { "type": "Secret Keyword", "filename": "pkg/models/operator.go", "hashed_secret": "f4e7a8740db0b7a0bfd8e63077261475f61fc2a6", "is_verified": false, - "line_number": 71 + "line_number": 74 }, { "type": "Secret Keyword", "filename": "pkg/models/operator.go", "hashed_secret": "7d4e4f654101e1514e18672295dfd53b64e7e5ee", "is_verified": false, - "line_number": 77 + "line_number": 80 }, { "type": "Secret Keyword", "filename": "pkg/models/operator.go", "hashed_secret": "e5e9fa1ba31ecd1ae84f75caaa474f3a663f05f4", "is_verified": false, - "line_number": 122 + "line_number": 125 }, { "type": "Secret Keyword", "filename": "pkg/models/operator.go", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 150 + "line_number": 153 }, { "type": "Secret Keyword", "filename": "pkg/models/operator.go", "hashed_secret": "d65d45369e8aef106a8ca1c3bad151ad24163494", "is_verified": false, - "line_number": 180 + "line_number": 183 }, { "type": "Secret Keyword", "filename": "pkg/models/operator.go", "hashed_secret": "638724dcc0799a22cc4adce12434fcac73c8af58", "is_verified": false, - "line_number": 181 + "line_number": 184 }, { "type": "Secret Keyword", "filename": "pkg/models/operator.go", "hashed_secret": "4fe486f255f36f8787d5c5cc1185e3d5d5c91c03", "is_verified": false, - "line_number": 182 + "line_number": 185 }, { "type": "Secret Keyword", "filename": "pkg/models/operator.go", "hashed_secret": "2331919a92cbb5c2d530947171fa5e1a1415af2f", "is_verified": false, - "line_number": 183 + "line_number": 186 } ], "scripts/cloud-init-secret.yaml": [ @@ -1135,5 +1135,5 @@ } ] }, - "generated_at": "2024-02-05T09:41:58Z" + "generated_at": "2024-02-06T13:59:12Z" } diff --git a/apis/clusters/v1beta1/postgresql_types.go b/apis/clusters/v1beta1/postgresql_types.go index 42f331a0b..794e79741 100644 --- a/apis/clusters/v1beta1/postgresql_types.go +++ b/apis/clusters/v1beta1/postgresql_types.go @@ -24,7 +24,6 @@ import ( k8scorev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" @@ -339,28 +338,6 @@ func (pdc *PgDataCentre) ArePGBouncersEqual(iPGBs []*PgBouncer) bool { return true } -func (pg *PostgreSQL) GetUserSecretName(ctx context.Context, k8sClient client.Client) (string, error) { - var err error - - labelsToQuery := fmt.Sprintf("%s=%s", models.ClusterIDLabel, pg.Status.ID) - selector, err := labels.Parse(labelsToQuery) - if err != nil { - return "", err - } - - userSecretList := &k8scorev1.SecretList{} - err = k8sClient.List(ctx, userSecretList, &client.ListOptions{LabelSelector: selector}) - if err != nil { - return "", err - } - - if len(userSecretList.Items) == 0 { - return "", nil - } - - return userSecretList.Items[0].Name, nil -} - func (pg *PostgreSQL) NewUserSecret(defaultUserPassword string) *k8scorev1.Secret { return &k8scorev1.Secret{ TypeMeta: metav1.TypeMeta{ diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 878ad4866..cd3a10f6e 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -12,6 +12,7 @@ metadata: namespace: system labels: control-plane: controller-manager + app: instaclustr-k8s-operator spec: selector: matchLabels: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d82f75c40..a9be51455 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -98,6 +98,8 @@ rules: verbs: - get - list + - patch + - update - watch - apiGroups: - cdi.kubevirt.io diff --git a/controllers/clusters/cadence_controller.go b/controllers/clusters/cadence_controller.go index 42a3a1473..02da4df81 100644 --- a/controllers/clusters/cadence_controller.go +++ b/controllers/clusters/cadence_controller.go @@ -61,6 +61,7 @@ type CadenceReconciler struct { //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=cadences/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=events,verbs=create //+kubebuilder:rbac:groups="",resources=endpoints,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list;update;patch;watch //+kubebuilder:rbac:groups=cdi.kubevirt.io,resources=datavolumes,verbs=get;list;watch;create;update;patch;delete;deletecollection //+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachines,verbs=get;list;watch;create;update;patch;delete;deletecollection //+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances,verbs=get;list;watch;create;update;patch;delete;deletecollection diff --git a/main.go b/main.go index b7ef5dbc5..eb4ca6466 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "context" "flag" "os" "time" @@ -42,6 +43,7 @@ import ( "github.com/instaclustr/operator/pkg/instaclustr" "github.com/instaclustr/operator/pkg/ratelimiter" "github.com/instaclustr/operator/pkg/scheduler" + "github.com/instaclustr/operator/pkg/upgradecheck" //+kubebuilder:scaffold:imports ) @@ -121,6 +123,17 @@ func main() { s := scheduler.NewScheduler(log.Log.WithValues("component", "scheduler")) + // TODO: take this variable from helm env + autoUpgradeEnabled := false + if autoUpgradeEnabled { + setupLog.Info("auto upgrade operator is enabled") + + err = upgradecheck.StartUpgradeCheckJob(context.TODO(), mgr.GetClient(), s) + if err != nil { + setupLog.Error(err, "unable to start operator upgrade check job") + } + } + eventRecorder := mgr.GetEventRecorderFor("instaclustr-operator") if err = (&clusterscontrollers.CassandraReconciler{ diff --git a/pkg/models/operator.go b/pkg/models/operator.go index 0a1e3efae..3a161e258 100644 --- a/pkg/models/operator.go +++ b/pkg/models/operator.go @@ -23,6 +23,9 @@ import ( ) const ( + InstOperatorDeploymentLabel = "instaclustr-k8s-operator" + InstOperatorContainerName = "manager" + ResourceStateAnnotation = "instaclustr.com/resourceState" ClusterDeletionAnnotation = "instaclustr.com/clusterDeletion" ExternalChangesAnnotation = "instaclustr.com/externalChanges" diff --git a/pkg/models/upgrade_check.go b/pkg/models/upgrade_check.go new file mode 100644 index 000000000..d5b9a35f4 --- /dev/null +++ b/pkg/models/upgrade_check.go @@ -0,0 +1,11 @@ +package models + +const ( + OperatorUpgradeChecker = "operatorUpgradeChecker" +) + +type DockerTagsResponse struct { + Results []struct { + Name string `json:"name"` + } `json:"results"` +} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index a8622ed43..2c0d5cbd2 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -33,6 +33,8 @@ const ( BackupsChecker = "backupsChecker" UserCreator = "userCreator" OnPremisesIPsChecker = "onPremisesIPsChecker" + + AutoUpgradeCheckInterval = 24 * time.Hour ) type Job func() error diff --git a/pkg/upgradecheck/upgrade_check.go b/pkg/upgradecheck/upgrade_check.go new file mode 100644 index 000000000..5db38d66c --- /dev/null +++ b/pkg/upgradecheck/upgrade_check.go @@ -0,0 +1,146 @@ +package upgradecheck + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/go-logr/logr" + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/instaclustr/operator/pkg/models" + "github.com/instaclustr/operator/pkg/scheduler" +) + +func StartUpgradeCheckJob(ctx context.Context, client client.Client, s scheduler.Interface) error { + job := newUpgradeCheckJob(ctx, client) + return s.ScheduleJob(models.OperatorUpgradeChecker, scheduler.AutoUpgradeCheckInterval, job) +} + +func newUpgradeCheckJob(ctx context.Context, client client.Client) scheduler.Job { + l := log.FromContext(ctx, "components", "UpgradeCheckJob") + + return func() error { + // TODO: change from dockerhub to custom endpoint + latestTag, err := getLatestDockerImageTag("icoperator/instaclustr-operator") + if err != nil { + return fmt.Errorf("unable to get latest docker image tag: %w", err) + } + + err = updateImageTagIfNeeded(ctx, l, client, latestTag) + if err != nil { + return fmt.Errorf("unable to update current image tag: %w", err) + } + + return nil + } +} + +func getLatestDockerImageTag(imageName string) (string, error) { + url := fmt.Sprintf("https://registry.hub.docker.com/v2/repositories/%s/tags?page_size=1", imageName) + + resp, err := http.Get(url) + if err != nil { + return "", err + } + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + var tagsResponse models.DockerTagsResponse + err = json.Unmarshal(body, &tagsResponse) + if err != nil { + return "", err + } + + if len(tagsResponse.Results) == 0 { + return "", fmt.Errorf("no tags found for image %s", imageName) + } + + return tagsResponse.Results[0].Name, nil +} + +func updateImageTagIfNeeded(ctx context.Context, l logr.Logger, mgrClient client.Client, latestTag string) error { + instDeployment, err := findInstOperatorDeployment(ctx, mgrClient) + if err != nil { + return err + } + + container, err := findContainer(instDeployment, models.InstOperatorContainerName) + if err != nil { + return err + } + + currentImageTag, err := getCurrentImageTag(container.Image) + if err != nil { + return err + } + + if currentImageTag != latestTag { + // TODO: add rollback in error case and check health status before update + if err := updateContainerImage(ctx, mgrClient, instDeployment, container, latestTag); err != nil { + return fmt.Errorf("cannot update latest docker image: %w", err) + } + + l.Info("Operator has been updated to the latest version", "old version", currentImageTag, "new version", latestTag) + } else { + l.Info("The operator is already up to date", "current version", currentImageTag) + } + + return nil +} + +func findInstOperatorDeployment(ctx context.Context, mgrClient client.Client) (*v1.Deployment, error) { + labelsToQuery := fmt.Sprintf("%s=%s", "app", models.InstOperatorDeploymentLabel) + selector, err := labels.Parse(labelsToQuery) + if err != nil { + return nil, fmt.Errorf("cannot parse label selector: %w", err) + } + + deploymentList := &v1.DeploymentList{} + if err := mgrClient.List(ctx, deploymentList, &client.ListOptions{LabelSelector: selector}); err != nil { + return nil, fmt.Errorf("cannot get instaclustr deployment: %w", err) + } + + if len(deploymentList.Items) != 1 { + return nil, fmt.Errorf("expected exactly one deployment, found %d", len(deploymentList.Items)) + } + + return &deploymentList.Items[0], nil +} + +func findContainer(deployment *v1.Deployment, containerName string) (*corev1.Container, error) { + for _, c := range deployment.Spec.Template.Spec.Containers { + if c.Name == containerName { + return &c, nil + } + } + + return nil, fmt.Errorf("cannot find container %s in the deployment", containerName) +} + +func getCurrentImageTag(image string) (string, error) { + parts := strings.Split(image, ":") + if len(parts) < 2 { + return "", fmt.Errorf("cannot find tag in the image") + } + + return parts[1], nil +} + +func updateContainerImage(ctx context.Context, mgrClient client.Client, deployment *v1.Deployment, container *corev1.Container, newTag string) error { + imageParts := strings.Split(container.Image, ":") + container.Image = fmt.Sprintf("%s:%s", imageParts[0], newTag) + + return mgrClient.Update(ctx, deployment) +}