diff --git a/cmd/main.go b/cmd/main.go index 344d936e..1ce55a98 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -167,6 +167,9 @@ func main() { Client: client, Scheme: mgr.GetScheme(), Log: ctrl.Log.WithName("controller").WithName("AerospikeBackupService"), + Recorder: eventBroadcaster.NewRecorder( + mgr.GetScheme(), v1.EventSource{Component: "aerospikeBackupService-controller"}, + ), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "AerospikeBackupService") os.Exit(1) @@ -181,6 +184,9 @@ func main() { Client: client, Scheme: mgr.GetScheme(), Log: ctrl.Log.WithName("controller").WithName("AerospikeBackup"), + Recorder: eventBroadcaster.NewRecorder( + mgr.GetScheme(), v1.EventSource{Component: "aerospikeBackup-controller"}, + ), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "AerospikeBackup") os.Exit(1) @@ -195,6 +201,9 @@ func main() { Client: client, Scheme: mgr.GetScheme(), Log: ctrl.Log.WithName("controller").WithName("AerospikeRestore"), + Recorder: eventBroadcaster.NewRecorder( + mgr.GetScheme(), v1.EventSource{Component: "aerospikeRestore-controller"}, + ), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "AerospikeRestore") os.Exit(1) diff --git a/internal/controller/backup-service/aerospikebackupservice_controller.go b/internal/controller/backup-service/aerospikebackupservice_controller.go index e9965e4a..96a9341c 100644 --- a/internal/controller/backup-service/aerospikebackupservice_controller.go +++ b/internal/controller/backup-service/aerospikebackupservice_controller.go @@ -20,8 +20,10 @@ import ( "context" "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -36,7 +38,8 @@ import ( type AerospikeBackupServiceReconciler struct { Scheme *k8sruntime.Scheme client.Client - Log logr.Logger + Recorder record.EventRecorder + Log logr.Logger } //nolint:lll // for readability @@ -56,6 +59,16 @@ func (r *AerospikeBackupServiceReconciler) Reconcile(_ context.Context, request aeroBackupService := &asdbv1beta1.AerospikeBackupService{} if err := r.Client.Get(context.TODO(), request.NamespacedName, aeroBackupService); err != nil { if errors.IsNotFound(err) { + log.Info("Deleted AerospikeBackupService") + + aeroBackupService.Namespace = request.Namespace + aeroBackupService.Name = request.Name + r.Recorder.Eventf( + aeroBackupService, corev1.EventTypeNormal, "Deleted", + "Deleted AerospikeBackupService %s/%s", aeroBackupService.Namespace, + aeroBackupService.Name, + ) + // Request object not found, could have been deleted after Reconcile request. return reconcile.Result{}, nil } @@ -68,6 +81,7 @@ func (r *AerospikeBackupServiceReconciler) Reconcile(_ context.Context, request Client: r.Client, Log: log, Scheme: r.Scheme, + Recorder: r.Recorder, } return cr.Reconcile() diff --git a/internal/controller/backup-service/reconciler.go b/internal/controller/backup-service/reconciler.go index 258ad2ec..809f60d8 100644 --- a/internal/controller/backup-service/reconciler.go +++ b/internal/controller/backup-service/reconciler.go @@ -63,6 +63,13 @@ func (r *SingleBackupServiceReconciler) Reconcile() (result ctrl.Result, recErr }() if !r.aeroBackupService.ObjectMeta.DeletionTimestamp.IsZero() { + r.Log.Info("Deleted AerospikeBackupService") + r.Recorder.Eventf( + r.aeroBackupService, corev1.EventTypeNormal, "Deleted", + "Deleted AerospikeBackupService %s/%s", r.aeroBackupService.Namespace, + r.aeroBackupService.Name, + ) + // Stop reconciliation as the Aerospike Backup service is being deleted return reconcile.Result{}, nil } @@ -73,21 +80,44 @@ func (r *SingleBackupServiceReconciler) Reconcile() (result ctrl.Result, recErr } if err := r.reconcileConfigMap(); err != nil { + r.Log.Error(err, "Failed to reconcile config map") + r.Recorder.Eventf(r.aeroBackupService, corev1.EventTypeWarning, + "ConfigMapReconcileFailed", "Failed to reconcile config map %s/%s", + r.aeroBackupService.Namespace, r.aeroBackupService.Name) + recErr = err + return ctrl.Result{}, err } if err := r.reconcileDeployment(); err != nil { + r.Log.Error(err, "Failed to reconcile deployment") + r.Recorder.Eventf(r.aeroBackupService, corev1.EventTypeWarning, + "DeploymentReconcileFailed", "Failed to reconcile deployment %s/%s", + r.aeroBackupService.Namespace, r.aeroBackupService.Name) + recErr = err + return ctrl.Result{}, err } if err := r.reconcileService(); err != nil { + r.Log.Error(err, "Failed to reconcile service") + r.Recorder.Eventf(r.aeroBackupService, corev1.EventTypeWarning, + "ServiceReconcileFailed", "Failed to reconcile service %s/%s", + r.aeroBackupService.Namespace, r.aeroBackupService.Name) + recErr = err + return ctrl.Result{}, err } if err := r.updateStatus(); err != nil { + r.Log.Error(err, "Failed to update status") + r.Recorder.Eventf(r.aeroBackupService, corev1.EventTypeWarning, + "StatusUpdateFailed", "Failed to update AerospikeBackupService status %s/%s", + r.aeroBackupService.Namespace, r.aeroBackupService.Name) + return ctrl.Result{}, err } @@ -107,7 +137,7 @@ func (r *SingleBackupServiceReconciler) reconcileConfigMap() error { return err } - r.Log.Info("Create Backup Service ConfigMap", + r.Log.Info("Creating Backup Service ConfigMap", "name", getBackupServiceName(r.aeroBackupService)) cm = &corev1.ConfigMap{ @@ -136,14 +166,16 @@ func (r *SingleBackupServiceReconciler) reconcileConfigMap() error { ) } - r.Log.Info("Created new Backup Service ConfigMap", + r.Log.Info("Created Backup Service ConfigMap", "name", getBackupServiceName(r.aeroBackupService)) + r.Recorder.Eventf(r.aeroBackupService, corev1.EventTypeNormal, "ConfigMapCreated", + "Created Backup Service ConfigMap %s/%s", r.aeroBackupService.Namespace, r.aeroBackupService.Name) return nil } r.Log.Info( - "ConfigMap already exist. Updating existing ConfigMap if required", + "Backup Service ConfigMap already exist. Updating existing ConfigMap if required", "name", getBackupServiceName(r.aeroBackupService), ) @@ -183,6 +215,8 @@ func (r *SingleBackupServiceReconciler) reconcileConfigMap() error { r.Log.Info("Updated Backup Service ConfigMap", "name", getBackupServiceName(r.aeroBackupService)) + r.Recorder.Eventf(r.aeroBackupService, corev1.EventTypeNormal, "ConfigMapUpdated", + "Updated Backup Service ConfigMap %s/%s", r.aeroBackupService.Namespace, r.aeroBackupService.Name) return nil } @@ -207,7 +241,7 @@ func (r *SingleBackupServiceReconciler) reconcileDeployment() error { return err } - r.Log.Info("Create Backup Service deployment", + r.Log.Info("Creating Backup Service deployment", "name", getBackupServiceName(r.aeroBackupService)) deployment, err := r.getDeploymentObject() @@ -228,6 +262,11 @@ func (r *SingleBackupServiceReconciler) reconcileDeployment() error { return fmt.Errorf("failed to deploy Backup service deployment: %v", err) } + r.Log.Info("Created Backup Service deployment", + "name", getBackupServiceName(r.aeroBackupService)) + r.Recorder.Eventf(r.aeroBackupService, corev1.EventTypeNormal, "DeploymentCreated", + "Created Backup Service Deployment %s/%s", r.aeroBackupService.Namespace, r.aeroBackupService.Name) + return r.waitForDeploymentToBeReady() } @@ -249,6 +288,11 @@ func (r *SingleBackupServiceReconciler) reconcileDeployment() error { return fmt.Errorf("failed to update Backup service deployment: %v", err) } + r.Log.Info("Updated Backup Service deployment", + "name", getBackupServiceName(r.aeroBackupService)) + r.Recorder.Eventf(r.aeroBackupService, corev1.EventTypeNormal, "DeploymentUpdated", + "Updated Backup Service Deployment %s/%s", r.aeroBackupService.Namespace, r.aeroBackupService.Name) + if oldResourceVersion != deploy.ResourceVersion { r.Log.Info("Deployment spec is updated, will result in rolling restart") return r.waitForDeploymentToBeReady() @@ -457,7 +501,7 @@ func (r *SingleBackupServiceReconciler) reconcileService() error { return err } - r.Log.Info("Create Backup Service", + r.Log.Info("Creating Backup Service", "name", getBackupServiceName(r.aeroBackupService)) svc, err := r.getServiceObject() @@ -478,6 +522,11 @@ func (r *SingleBackupServiceReconciler) reconcileService() error { return fmt.Errorf("failed to create Backup Service: %v", err) } + r.Log.Info("Created Backup Service", + "name", getBackupServiceName(r.aeroBackupService)) + r.Recorder.Eventf(r.aeroBackupService, corev1.EventTypeNormal, "ServiceCreated", + "Created Backup Service %s/%s", r.aeroBackupService.Namespace, r.aeroBackupService.Name) + return nil } @@ -498,6 +547,8 @@ func (r *SingleBackupServiceReconciler) reconcileService() error { } r.Log.Info("Updated Backup Service", "name", getBackupServiceName(r.aeroBackupService)) + r.Recorder.Eventf(r.aeroBackupService, corev1.EventTypeNormal, "ServiceUpdated", + "Updated Backup Service %s/%s", r.aeroBackupService.Namespace, r.aeroBackupService.Name) return nil } diff --git a/internal/controller/backup/aerospikebackup_controller.go b/internal/controller/backup/aerospikebackup_controller.go index 9310a3a4..658a238a 100644 --- a/internal/controller/backup/aerospikebackup_controller.go +++ b/internal/controller/backup/aerospikebackup_controller.go @@ -22,6 +22,7 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/errors" k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -37,8 +38,9 @@ const finalizerName = "asdb.aerospike.com/backup-finalizer" // AerospikeBackupReconciler reconciles a AerospikeBackup object type AerospikeBackupReconciler struct { client.Client - Scheme *k8sruntime.Scheme - Log logr.Logger + Scheme *k8sruntime.Scheme + Recorder record.EventRecorder + Log logr.Logger } //+kubebuilder:rbac:groups=asdb.aerospike.com,resources=aerospikebackups,verbs=get;list;watch;create;update;patch;delete @@ -68,6 +70,7 @@ func (r *AerospikeBackupReconciler) Reconcile(_ context.Context, request ctrl.Re Client: r.Client, Log: log, Scheme: r.Scheme, + Recorder: r.Recorder, } return cr.Reconcile() diff --git a/internal/controller/backup/reconciler.go b/internal/controller/backup/reconciler.go index fbe48454..6edeafd6 100644 --- a/internal/controller/backup/reconciler.go +++ b/internal/controller/backup/reconciler.go @@ -37,11 +37,18 @@ type SingleBackupReconciler struct { func (r *SingleBackupReconciler) Reconcile() (result ctrl.Result, recErr error) { // Check DeletionTimestamp to see if the backup is being deleted if !r.aeroBackup.ObjectMeta.DeletionTimestamp.IsZero() { + r.Log.Info("Deleting AerospikeBackup") + if err := r.removeFinalizer(finalizerName); err != nil { r.Log.Error(err, "Failed to remove finalizer") return reconcile.Result{}, err } + r.Recorder.Eventf( + r.aeroBackup, corev1.EventTypeNormal, "Deleted", + "Deleted AerospikeBackup %s/%s", r.aeroBackup.Namespace, + r.aeroBackup.Name, + ) // Stop reconciliation as the backup is being deleted return reconcile.Result{}, nil } @@ -54,16 +61,28 @@ func (r *SingleBackupReconciler) Reconcile() (result ctrl.Result, recErr error) if err := r.reconcileConfigMap(); err != nil { r.Log.Error(err, "Failed to reconcile config map") + r.Recorder.Eventf(r.aeroBackup, corev1.EventTypeWarning, + "ConfigMapReconcileFailed", "Failed to reconcile config map %s", + r.aeroBackup.Spec.BackupService.String()) + return reconcile.Result{}, err } if err := r.reconcileBackup(); err != nil { r.Log.Error(err, "Failed to reconcile backup") + r.Recorder.Eventf(r.aeroBackup, corev1.EventTypeWarning, + "BackupReconcileFailed", "Failed to reconcile backup %s/%s", + r.aeroBackup.Namespace, r.aeroBackup.Name) + return reconcile.Result{}, err } if err := r.updateStatus(); err != nil { r.Log.Error(err, "Failed to update status") + r.Recorder.Eventf(r.aeroBackup, corev1.EventTypeWarning, + "StatusUpdateFailed", "Failed to update AerospikeBackup status %s/%s", + r.aeroBackup.Namespace, r.aeroBackup.Name) + return reconcile.Result{}, err } @@ -198,6 +217,9 @@ func (r *SingleBackupReconciler) reconcileConfigMap() error { r.Log.Info("Updated Backup Service ConfigMap for Backup", "name", r.aeroBackup.Spec.BackupService.String(), ) + r.Recorder.Eventf(r.aeroBackup, corev1.EventTypeNormal, "ConfigMapUpdated", + "Updated Backup Service ConfigMap %s for Backup %s/%s", r.aeroBackup.Spec.BackupService.String(), + r.aeroBackup.Namespace, r.aeroBackup.Name) return nil } @@ -292,7 +314,7 @@ func (r *SingleBackupReconciler) scheduleOnDemandBackup() error { return nil } - r.Log.Info("Schedule on-demand backup", + r.Log.Info("Scheduling on-demand backup", "ID", r.aeroBackup.Spec.OnDemandBackups[0].ID, "routine", r.aeroBackup.Spec.OnDemandBackups[0].RoutineName) backupServiceClient, err := backup_service.GetBackupServiceClient(r.Client, &r.aeroBackup.Spec.BackupService) @@ -308,8 +330,10 @@ func (r *SingleBackupReconciler) scheduleOnDemandBackup() error { r.Log.Info("Scheduled on-demand backup", "ID", r.aeroBackup.Spec.OnDemandBackups[0].ID, "routine", r.aeroBackup.Spec.OnDemandBackups[0].RoutineName) + r.Recorder.Eventf(r.aeroBackup, corev1.EventTypeNormal, "BackupScheduled", + "Scheduled on-demand backup %s/%s", r.aeroBackup.Namespace, r.aeroBackup.Name) - r.Log.Info("Reconciled scheduled backup") + r.Log.Info("Reconciled on-demand backup") return nil } @@ -421,6 +445,8 @@ func (r *SingleBackupReconciler) reconcileScheduledBackup() error { } r.Log.Info("Reconciled scheduled backup") + r.Recorder.Eventf(r.aeroBackup, corev1.EventTypeNormal, "BackupReconciled", + "Reconciled scheduled backup %s/%s", r.aeroBackup.Namespace, r.aeroBackup.Name) return nil } diff --git a/internal/controller/restore/aerospikerestore_controller.go b/internal/controller/restore/aerospikerestore_controller.go index b622a683..8dabb236 100644 --- a/internal/controller/restore/aerospikerestore_controller.go +++ b/internal/controller/restore/aerospikerestore_controller.go @@ -20,23 +20,26 @@ import ( "context" "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" k8sRuntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" asdbv1beta1 "github.com/aerospike/aerospike-kubernetes-operator/api/v1beta1" "github.com/aerospike/aerospike-kubernetes-operator/internal/controller/common" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" ) // AerospikeRestoreReconciler reconciles a AerospikeRestore object type AerospikeRestoreReconciler struct { client.Client - Scheme *k8sRuntime.Scheme - Log logr.Logger + Scheme *k8sRuntime.Scheme + Recorder record.EventRecorder + Log logr.Logger } //nolint:lll // for readability @@ -55,6 +58,16 @@ func (r *AerospikeRestoreReconciler) Reconcile(_ context.Context, request ctrl.R aeroRestore := &asdbv1beta1.AerospikeRestore{} if err := r.Client.Get(context.TODO(), request.NamespacedName, aeroRestore); err != nil { if errors.IsNotFound(err) { + log.Info("Deleted AerospikeRestore") + + aeroRestore.Namespace = request.Namespace + aeroRestore.Name = request.Name + r.Recorder.Eventf( + aeroRestore, corev1.EventTypeNormal, "Deleted", + "Deleted AerospikeRestore %s/%s", aeroRestore.Namespace, + aeroRestore.Name, + ) + // Request object not found, could have been deleted after Reconcile request. return reconcile.Result{}, nil } @@ -67,6 +80,7 @@ func (r *AerospikeRestoreReconciler) Reconcile(_ context.Context, request ctrl.R Client: r.Client, Log: log, Scheme: r.Scheme, + Recorder: r.Recorder, } return cr.Reconcile() diff --git a/internal/controller/restore/reconciler.go b/internal/controller/restore/reconciler.go index 435cd994..30075efb 100644 --- a/internal/controller/restore/reconciler.go +++ b/internal/controller/restore/reconciler.go @@ -7,6 +7,7 @@ import ( "net/http" "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" k8sRuntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" @@ -31,6 +32,13 @@ type SingleRestoreReconciler struct { func (r *SingleRestoreReconciler) Reconcile() (result ctrl.Result, recErr error) { if !r.aeroRestore.ObjectMeta.DeletionTimestamp.IsZero() { + r.Log.Info("Deleted AerospikeRestore") + r.Recorder.Eventf( + r.aeroRestore, corev1.EventTypeNormal, "Deleted", + "Deleted AerospikeRestore %s/%s", r.aeroRestore.Namespace, + r.aeroRestore.Name, + ) + // Stop reconciliation as the Aerospike restore is being deleted return reconcile.Result{}, nil } @@ -41,6 +49,10 @@ func (r *SingleRestoreReconciler) Reconcile() (result ctrl.Result, recErr error) if res := r.reconcileRestore(); !res.IsSuccess { if res.Err != nil { + r.Log.Error(res.Err, "Failed to reconcile restore") + r.Recorder.Eventf(r.aeroRestore, corev1.EventTypeWarning, "RestoreReconcileFailed", + "Failed to reconcile restore %s/%s", r.aeroRestore.Namespace, r.aeroRestore.Name) + return res.Result, res.Err } @@ -48,6 +60,10 @@ func (r *SingleRestoreReconciler) Reconcile() (result ctrl.Result, recErr error) } if err := r.checkRestoreStatus(); err != nil { + r.Log.Error(err, "Failed to check restore status") + r.Recorder.Eventf(r.aeroRestore, corev1.EventTypeWarning, "RestoreStatusCheckFailed", + "Failed to check restore status %s/%s", r.aeroRestore.Namespace, r.aeroRestore.Name) + return ctrl.Result{}, err } @@ -55,6 +71,9 @@ func (r *SingleRestoreReconciler) Reconcile() (result ctrl.Result, recErr error) return ctrl.Result{RequeueAfter: r.aeroRestore.Spec.PollingPeriod.Duration}, nil } + r.Recorder.Eventf(r.aeroRestore, corev1.EventTypeNormal, "RestoreCompleted", + "Restore completed successfully %s/%s", r.aeroRestore.Namespace, r.aeroRestore.Name) + return ctrl.Result{}, nil } @@ -109,6 +128,9 @@ func (r *SingleRestoreReconciler) reconcileRestore() common.ReconcileResult { return common.ReconcileError(err) } + r.Recorder.Eventf(r.aeroRestore, corev1.EventTypeNormal, "RestoreTriggered", + "Triggered restore %s/%s", r.aeroRestore.Namespace, r.aeroRestore.Name) + r.aeroRestore.Status.JobID = jobID if err = r.Client.Status().Update(context.Background(), r.aeroRestore); err != nil {