Skip to content

Commit

Permalink
NodeReload controller was refactored
Browse files Browse the repository at this point in the history
  • Loading branch information
Bohdan Siryk authored and Bohdan Siryk committed Oct 6, 2023
1 parent 3d59a9a commit fac2810
Show file tree
Hide file tree
Showing 5 changed files with 162 additions and 54 deletions.
26 changes: 11 additions & 15 deletions apis/clusterresources/v1beta1/nodereload_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,11 @@ type NodeReloadSpec struct {

// NodeReloadStatus defines the observed state of NodeReload
type NodeReloadStatus struct {
NodeInProgress Node `json:"nodeInProgress,omitempty"`
NodeInProgress *Node `json:"nodeInProgress,omitempty"`
CurrentOperationStatus *Operation `json:"currentOperationStatus,omitempty"`
PendingNodes []*Node `json:"pendingNodes,omitempty"`
CompletedNodes []*Node `json:"completedNodes,omitempty"`
FailedNodes []*Node `json:"failedNodes,omitempty"`
}

type Node struct {
Expand Down Expand Up @@ -76,19 +79,12 @@ func init() {
SchemeBuilder.Register(&NodeReload{}, &NodeReloadList{})
}

func (nr *NodeReloadStatus) FromInstAPI(status *models.NodeReloadStatus) *NodeReloadStatus {
var nrStatus = &NodeReloadStatus{
NodeInProgress: Node{
ID: status.NodeID,
},
CurrentOperationStatus: &Operation{
OperationID: status.OperationID,
TimeCreated: status.TimeCreated,
TimeModified: status.TimeModified,
Status: status.Status,
Message: status.Message,
},
func (nr *NodeReloadStatus) UpdateCurrentOperationStatus(status *models.NodeReloadStatus) {
nr.CurrentOperationStatus = &Operation{
OperationID: status.OperationID,
TimeCreated: status.TimeCreated,
TimeModified: status.TimeModified,
Status: status.Status,
Message: status.Message,
}

return nrStatus
}
39 changes: 38 additions & 1 deletion apis/clusterresources/v1beta1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

27 changes: 27 additions & 0 deletions config/crd/bases/clusterresources.instaclustr.com_nodereloads.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,15 @@ spec:
status:
description: NodeReloadStatus defines the observed state of NodeReload
properties:
completedNodes:
items:
properties:
nodeID:
type: string
required:
- nodeID
type: object
type: array
currentOperationStatus:
properties:
message:
Expand All @@ -68,13 +77,31 @@ spec:
- timeCreated
- timeModified
type: object
failedNodes:
items:
properties:
nodeID:
type: string
required:
- nodeID
type: object
type: array
nodeInProgress:
properties:
nodeID:
type: string
required:
- nodeID
type: object
pendingNodes:
items:
properties:
nodeID:
type: string
required:
- nodeID
type: object
type: array
type: object
type: object
served: true
Expand Down
120 changes: 82 additions & 38 deletions controllers/clusterresources/nodereload_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package clusterresources

import (
"context"
"errors"

k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -28,6 +29,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"

"github.com/instaclustr/operator/apis/clusterresources/v1beta1"
"github.com/instaclustr/operator/pkg/instaclustr"
Expand All @@ -42,6 +44,10 @@ type NodeReloadReconciler struct {
EventRecorder record.EventRecorder
}

const (
NodeReloadOperationStatusCompleted = "COMPLETED"
)

//+kubebuilder:rbac:groups=clusterresources.instaclustr.com,resources=nodereloads,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=clusterresources.instaclustr.com,resources=nodereloads/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=clusterresources.instaclustr.com,resources=nodereloads/finalizers,verbs=update
Expand All @@ -66,63 +72,68 @@ func (r *NodeReloadReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return models.ReconcileRequeue, err
}

if len(nrs.Spec.Nodes) == 0 {
err = r.Client.Delete(ctx, nrs)
patch := nrs.NewPatch()
if len(nrs.Status.PendingNodes) == 0 && len(nrs.Status.CompletedNodes) == 0 && len(nrs.Status.FailedNodes) == 0 {
nrs.Status.PendingNodes = nrs.Spec.Nodes
err = r.Status().Patch(ctx, nrs, patch)
if err != nil {
l.Error(err,
"Cannot delete Node Reload resource from K8s cluster",
"Node Reload spec", nrs.Spec,
)
r.EventRecorder.Eventf(
nrs, models.Warning, models.DeletionFailed,
"Resource deletion is failed. Reason: %v",
err,
l.Error(err, "Cannot set pending nodes to the status")
r.EventRecorder.Event(nrs, models.Warning, models.PatchFailed,
"Cannot set pending nodes to the resource status",
)

return models.ReconcileRequeue, nil
}
}

if len(nrs.Status.PendingNodes) == 0 && nrs.Status.NodeInProgress == nil {
r.EventRecorder.Eventf(
nrs, models.Normal, models.DeletionStarted,
"Resource is deleted.",
nrs, models.Normal, models.UpdatedEvent,
"The controller has finished working",
)
l.Info(
"Nodes were reloaded, resource was deleted",
"Node Reload spec", nrs.Spec,
"The controller has finished working",
"completed nodes", nrs.Status.CompletedNodes,
"failed nodes", nrs.Status.FailedNodes,
)

return models.ExitReconcile, nil
}
patch := nrs.NewPatch()

if nrs.Status.NodeInProgress.ID == "" {
nodeInProgress := &v1beta1.Node{
ID: nrs.Spec.Nodes[len(nrs.Spec.Nodes)-1].ID,
}
nrs.Status.NodeInProgress.ID = nodeInProgress.ID
if nrs.Status.NodeInProgress == nil {
nodeInProgress := nrs.Status.PendingNodes[len(nrs.Status.PendingNodes)-1]

err = r.API.CreateNodeReload(nodeInProgress)
if err != nil {
if errors.Is(err, instaclustr.NotFound) {
return r.handleNodeNotFound(ctx, nodeInProgress, nrs)
}

l.Error(err,
"Cannot start Node Reload process",
"nodeID", nodeInProgress.ID,
)
r.EventRecorder.Eventf(
nrs, models.Warning, models.CreationFailed,
"Resource creation on the Instaclustr is failed. Reason: %v",
"Sending node reload request to the Instaclustr is failed. Reason: %v",
err,
)
return models.ReconcileRequeue, nil
}

r.EventRecorder.Eventf(
nrs, models.Normal, models.Created,
"Resource creation request is sent. Node ID: %s",
nrs.Status.NodeInProgress.ID,
"Node reload request is sent. Node ID: %s",
nodeInProgress.ID,
)

nrs.Status.PendingNodes = nrs.Status.PendingNodes[:len(nrs.Status.PendingNodes)-1]
nrs.Status.NodeInProgress = nodeInProgress
err = r.Status().Patch(ctx, nrs, patch)
if err != nil {
l.Error(err,
"Cannot patch Node Reload status",
"nodeID", nrs.Status.NodeInProgress,
"Cannot patch NodeInProgress and PendingNodes status",
"nodeID", nodeInProgress.ID,
)
r.EventRecorder.Eventf(
nrs, models.Warning, models.PatchFailed,
Expand All @@ -135,6 +146,10 @@ func (r *NodeReloadReconciler) Reconcile(ctx context.Context, req ctrl.Request)

nodeReloadStatus, err := r.API.GetNodeReloadStatus(nrs.Status.NodeInProgress.ID)
if err != nil {
if errors.Is(err, instaclustr.NotFound) {
return r.handleNodeNotFound(ctx, nrs.Status.NodeInProgress, nrs)
}

l.Error(err,
"Cannot get Node Reload status",
"nodeID", nrs.Status.NodeInProgress,
Expand All @@ -147,7 +162,7 @@ func (r *NodeReloadReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return models.ReconcileRequeue, nil
}

nrs.Status = *nrs.Status.FromInstAPI(nodeReloadStatus)
nrs.Status.UpdateCurrentOperationStatus(nodeReloadStatus)
err = r.Status().Patch(ctx, nrs, patch)
if err != nil {
l.Error(err,
Expand All @@ -162,15 +177,25 @@ func (r *NodeReloadReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return models.ReconcileRequeue, nil
}

if nrs.Status.CurrentOperationStatus.Status != "COMPLETED" {
if nrs.Status.CurrentOperationStatus.Status != NodeReloadOperationStatusCompleted {
l.Info("Node Reload operation is not completed yet, please wait a few minutes",
"nodeID", nrs.Status.NodeInProgress,
"status", nrs.Status,
)
return models.ReconcileRequeue, nil
}

nrs.Status.NodeInProgress.ID = ""
l.Info("The node has been successfully reloaded",
"Node ID", nrs.Status.NodeInProgress.ID,
)
r.EventRecorder.Eventf(nrs, models.Normal, models.UpdatedEvent,
"Node %s has been successfully reloaded", nrs.Status.NodeInProgress.ID,
)

nrs.Status.CompletedNodes = append(nrs.Status.CompletedNodes, nrs.Status.NodeInProgress)
nrs.Status.CurrentOperationStatus = nil
nrs.Status.NodeInProgress = nil

err = r.Status().Patch(ctx, nrs, patch)
if err != nil {
l.Error(err,
Expand All @@ -185,21 +210,40 @@ func (r *NodeReloadReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return models.ReconcileRequeue, nil
}

nrs.Spec.Nodes = nrs.Spec.Nodes[:len(nrs.Spec.Nodes)-1]
err = r.Patch(ctx, nrs, patch)
return reconcile.Result{Requeue: true}, nil
}

func (r *NodeReloadReconciler) handleNodeNotFound(ctx context.Context, node *v1beta1.Node, nrs *v1beta1.NodeReload) (reconcile.Result, error) {
l := log.FromContext(ctx)

patch := nrs.NewPatch()

if len(nrs.Status.PendingNodes) > 0 && nrs.Status.NodeInProgress == nil {
nrs.Status.PendingNodes = nrs.Status.PendingNodes[:len(nrs.Status.PendingNodes)-1]
}

nrs.Status.FailedNodes = append(nrs.Status.FailedNodes, node)
nrs.Status.CurrentOperationStatus = nil
nrs.Status.NodeInProgress = nil

err := r.Status().Patch(ctx, nrs, patch)
if err != nil {
l.Error(err, "Cannot patch Node Reload cluster",
"spec", nrs.Spec,
)
r.EventRecorder.Eventf(
nrs, models.Warning, models.PatchFailed,
"Resource patch is failed. Reason: %v",
err,
l.Error(err, "Cannot patch failed node")
r.EventRecorder.Event(nrs, models.Warning, models.PatchFailed,
"Cannot patch failed node",
)

return models.ReconcileRequeue, nil
}

return models.ExitReconcile, nil
l.Error(err, "Node is not found on Instaclustr",
"Node ID", node.ID,
)
r.EventRecorder.Eventf(nrs, models.Warning, models.FetchFailed,
"Node %s is not found on Instaclustr", node.ID,
)

return reconcile.Result{Requeue: true}, nil
}

// SetupWithManager sets up the controller with the Manager.
Expand Down
4 changes: 4 additions & 0 deletions pkg/instaclustr/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1476,6 +1476,10 @@ func (c *Client) CreateNodeReload(nr *clusterresourcesv1beta1.Node) error {
return err
}

if resp.StatusCode == http.StatusNotFound {
return NotFound
}

if resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("status code: %d, message: %s", resp.StatusCode, body)
}
Expand Down

0 comments on commit fac2810

Please sign in to comment.