From a30e54414bd0fa5031cd128091052b664deaf3b8 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Tue, 25 Apr 2023 11:54:26 +0600 Subject: [PATCH 01/20] support nodegroup deletion --- cmd/cloud/cluster.go | 2 +- cmd/cloud/nodegroup.go | 52 +++++++++++++-- cmd/cloud/nodegroup_flag.go | 18 +++++- internal/api/cluster.go | 62 ++++++++++++++++++ internal/provisioner/eks_provisioner.go | 82 ++++++++++++++++++++++-- internal/provisioner/kops_provisioner.go | 12 +++- internal/supervisor/cluster.go | 26 ++++++-- internal/supervisor/cluster_test.go | 8 ++- model/client.go | 16 +++++ model/cluster_states.go | 10 +++ model/eks_metadata.go | 19 ++++++ 11 files changed, 283 insertions(+), 24 deletions(-) diff --git a/cmd/cloud/cluster.go b/cmd/cloud/cluster.go index 32a5d2690..07084beb0 100644 --- a/cmd/cloud/cluster.go +++ b/cmd/cloud/cluster.go @@ -40,7 +40,7 @@ func newCmdCluster() *cobra.Command { cmd.AddCommand(newCmdClusterUpdate()) cmd.AddCommand(newCmdClusterUpgrade()) cmd.AddCommand(newCmdClusterResize()) - cmd.AddCommand(newCmdClusterNodegroups()) + cmd.AddCommand(newCmdClusterNodegroup()) cmd.AddCommand(newCmdClusterDelete()) cmd.AddCommand(newCmdClusterGet()) cmd.AddCommand(newCmdClusterList()) diff --git a/cmd/cloud/nodegroup.go b/cmd/cloud/nodegroup.go index 58bfe64dc..ef1f22ce8 100644 --- a/cmd/cloud/nodegroup.go +++ b/cmd/cloud/nodegroup.go @@ -13,19 +13,20 @@ import ( "github.com/spf13/cobra" ) -func newCmdClusterNodegroups() *cobra.Command { +func newCmdClusterNodegroup() *cobra.Command { cmd := &cobra.Command{ - Use: "nodegroups", + Use: "nodegroup", Short: "Manipulate nodegroups of an existing cluster.", } - cmd.AddCommand(newCmdClusterNodegroupsAdd()) + cmd.AddCommand(newCmdClusterNodegroupCreate()) + cmd.AddCommand(newCmdClusterNodegroupDelete()) return cmd } -func newCmdClusterNodegroupsAdd() *cobra.Command { - var flags clusterNodegroupsAddFlags +func newCmdClusterNodegroupCreate() *cobra.Command { + var flags clusterNodegroupsCreateFlags cmd := &cobra.Command{ Use: "create", Short: "Create new nodegroups in an existing cluster.", @@ -43,7 +44,7 @@ func newCmdClusterNodegroupsAdd() *cobra.Command { return cmd } -func addNodegroup(flags clusterNodegroupsAddFlags) error { +func addNodegroup(flags clusterNodegroupsCreateFlags) error { client := model.NewClient(flags.serverAddress) if len(flags.nodegroups) == 0 { @@ -87,3 +88,42 @@ func addNodegroup(flags clusterNodegroupsAddFlags) error { return nil } + +func newCmdClusterNodegroupDelete() *cobra.Command { + var flags clusterNodegroupDeleteFlags + + cmd := &cobra.Command{ + Use: "delete", + Short: "Delete a nodegroup from an existing cluster.", + RunE: func(command *cobra.Command, args []string) error { + command.SilenceUsage = true + return deleteNodegroup(flags) + }, + PreRun: func(cmd *cobra.Command, args []string) { + flags.clusterFlags.addFlags(cmd) + }, + } + + flags.addFlags(cmd) + + return cmd +} + +func deleteNodegroup(flags clusterNodegroupDeleteFlags) error { + client := model.NewClient(flags.serverAddress) + + if len(flags.nodegroup) == 0 { + return fmt.Errorf("nodegroup must be provided") + } + + cluster, err := client.DeleteNodegroup(flags.clusterID, flags.nodegroup) + if err != nil { + return errors.Wrap(err, "failed to delete nodegroups") + } + + if err = printJSON(cluster); err != nil { + return errors.Wrap(err, "failed to print cluster response") + } + + return nil +} diff --git a/cmd/cloud/nodegroup_flag.go b/cmd/cloud/nodegroup_flag.go index 537afb466..059e6fa77 100644 --- a/cmd/cloud/nodegroup_flag.go +++ b/cmd/cloud/nodegroup_flag.go @@ -6,7 +6,7 @@ package main import "github.com/spf13/cobra" -type clusterNodegroupsAddFlags struct { +type clusterNodegroupsCreateFlags struct { clusterFlags clusterID string nodegroups map[string]string @@ -14,7 +14,7 @@ type clusterNodegroupsAddFlags struct { nodegroupsWithSecurityGroup []string } -func (flags *clusterNodegroupsAddFlags) addFlags(command *cobra.Command) { +func (flags *clusterNodegroupsCreateFlags) addFlags(command *cobra.Command) { command.Flags().StringVar(&flags.clusterID, "cluster", "", "The id of the cluster to be modified.") command.Flags().StringToStringVar(&flags.nodegroups, "nodegroups", nil, "Additional nodegroups to create. The key is the name of the nodegroup and the value is the size constant.") command.Flags().StringSliceVar(&flags.nodegroupsWithPublicSubnet, "nodegroups-with-public-subnet", nil, "Nodegroups to create with public subnet. The value is the name of the nodegroup.") @@ -23,3 +23,17 @@ func (flags *clusterNodegroupsAddFlags) addFlags(command *cobra.Command) { _ = command.MarkFlagRequired("cluster") _ = command.MarkFlagRequired("nodegroups") } + +type clusterNodegroupDeleteFlags struct { + clusterFlags + clusterID string + nodegroup string +} + +func (flags *clusterNodegroupDeleteFlags) addFlags(command *cobra.Command) { + command.Flags().StringVar(&flags.clusterID, "cluster", "", "The id of the cluster to be modified.") + command.Flags().StringVar(&flags.nodegroup, "nodegroup", "", "The name of the nodegroup to delete.") + + _ = command.MarkFlagRequired("cluster") + _ = command.MarkFlagRequired("nodegroup") +} diff --git a/internal/api/cluster.go b/internal/api/cluster.go index 28cd98bde..fdb4035e5 100644 --- a/internal/api/cluster.go +++ b/internal/api/cluster.go @@ -34,6 +34,7 @@ func initCluster(apiRouter *mux.Router, context *Context) { clusterRouter.Handle("/annotations", addContext(handleAddClusterAnnotations)).Methods("POST") clusterRouter.Handle("/annotation/{annotation-name}", addContext(handleDeleteClusterAnnotation)).Methods("DELETE") clusterRouter.Handle("/nodegroups", addContext(handleCreateNodegroups)).Methods("POST") + clusterRouter.Handle("/nodegroup/{nodegroup}", addContext(handleDeleteNodegroup)).Methods("DELETE") clusterRouter.Handle("", addContext(handleDeleteCluster)).Methods("DELETE") } @@ -496,6 +497,67 @@ func handleCreateNodegroups(c *Context, w http.ResponseWriter, r *http.Request) outputJSON(c, w, clusterDTO) } +// handleDeleteNodegroup responds to DELETE /api/cluster/{cluster}/nodegroup/{nodegroup}, deleting +// the requested nodegroup. +func handleDeleteNodegroup(c *Context, w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + clusterID := vars["cluster"] + nodegroup := vars["nodegroup"] + c.Logger = c.Logger.WithField("cluster", clusterID) + + newState := model.ClusterStateNodegroupDeletionRequested + + clusterDTO, status, unlockOnce := getClusterForTransition(c, clusterID, newState) + if status != 0 { + c.Logger.Debug("Cluster is not in a valid state for nodegroup deletion") + w.WriteHeader(status) + return + } + defer unlockOnce() + + if clusterDTO.Provisioner == model.ProvisionerKops { + c.Logger.Debug("Deleting nodegroup for Kops cluster is not supported") + w.WriteHeader(http.StatusBadRequest) + return + } + + if clusterDTO.Provisioner == model.ProvisionerEKS { + err := clusterDTO.ProvisionerMetadataEKS.ValidateNodegroupDeleteRequest(nodegroup) + if err != nil { + c.Logger.WithError(err).Error("failed to validate nodegroup delete request") + w.WriteHeader(http.StatusBadRequest) + return + } + + // Set Nodegroup in change request + clusterDTO.ProvisionerMetadataEKS.ApplyNodegroupDeleteRequest(nodegroup) + } + + oldState := clusterDTO.State + + clusterDTO.State = newState + err := c.Store.UpdateCluster(clusterDTO.Cluster) + if err != nil { + c.Logger.WithError(err).Error("failed to update cluster") + w.WriteHeader(http.StatusInternalServerError) + return + } + + if oldState != newState { + err = c.EventProducer.ProduceClusterStateChangeEvent(clusterDTO.Cluster, oldState) + if err != nil { + c.Logger.WithError(err).Error("failed to create cluster state change event") + } + } + + unlockOnce() + _ = c.Supervisor.Do() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + outputJSON(c, w, clusterDTO) +} + // handleDeleteCluster responds to DELETE /api/cluster/{cluster}, beginning the process of // deleting the cluster. func handleDeleteCluster(c *Context, w http.ResponseWriter, r *http.Request) { diff --git a/internal/provisioner/eks_provisioner.go b/internal/provisioner/eks_provisioner.go index 7599dba2c..101bac8c1 100644 --- a/internal/provisioner/eks_provisioner.go +++ b/internal/provisioner/eks_provisioner.go @@ -248,8 +248,8 @@ func (provisioner *EKSProvisioner) prepareLaunchTemplate(cluster *model.Cluster, return nil } -// CreateNodes creates the EKS nodes. -func (provisioner *EKSProvisioner) CreateNodes(cluster *model.Cluster) error { +// CreateNodegroups creates the EKS nodegroups. +func (provisioner *EKSProvisioner) CreateNodegroups(cluster *model.Cluster) error { logger := provisioner.logger.WithField("cluster", cluster.ID) eksMetadata := cluster.ProvisionerMetadataEKS @@ -300,8 +300,8 @@ func (provisioner *EKSProvisioner) CreateNodes(cluster *model.Cluster) error { return nil } -// CheckNodesCreated provisions EKS cluster. -func (provisioner *EKSProvisioner) CheckNodesCreated(cluster *model.Cluster) (bool, error) { +// CheckNodegroupsCreated checks if the EKS nodegroups are created. +func (provisioner *EKSProvisioner) CheckNodegroupsCreated(cluster *model.Cluster) (bool, error) { logger := provisioner.logger.WithField("cluster", cluster.ID) eksMetadata := cluster.ProvisionerMetadataEKS @@ -361,6 +361,80 @@ func (provisioner *EKSProvisioner) CheckNodesCreated(cluster *model.Cluster) (bo return true, nil } +// DeleteNodegroup deletes the EKS nodegroup. +func (provisioner *EKSProvisioner) DeleteNodegroup(cluster *model.Cluster) error { + logger := provisioner.logger.WithField("cluster", cluster.ID) + + eksMetadata := cluster.ProvisionerMetadataEKS + if eksMetadata == nil { + return errors.New("error: EKS metadata not set when deleting EKS NodeGroup") + } + + if eksMetadata.NodeGroups == nil || len(eksMetadata.NodeGroups) == 0 { + logger.Info("No EKS NodeGroup available to delete") + return nil + } + + nodeGroups := make(map[string]model.NodeGroupMetadata) + if eksMetadata.ChangeRequest != nil { + for ng := range eksMetadata.ChangeRequest.NodeGroups { + if metadata, found := eksMetadata.NodeGroups[ng]; found { + nodeGroups[ng] = metadata + } else { + logger.Warnf("EKS NodeGroup for %s not found in cluster metadata", ng) + } + } + } + + var wg sync.WaitGroup + var errOccurred bool + + for ng, meta := range nodeGroups { + wg.Add(1) + go func(ngPrefix string, ngMetadata model.NodeGroupMetadata) { + defer wg.Done() + + err := provisioner.awsClient.EnsureEKSNodeGroupDeleted(eksMetadata.Name, ngMetadata.Name) + if err != nil { + logger.WithError(err).Errorf("failed to delete EKS NodeGroup %s", ngMetadata.Name) + errOccurred = true + return + } + + wait := 600 + logger.Infof("Waiting up to %d seconds for NodeGroup %s to be deleted...", wait, ngMetadata.Name) + err = provisioner.awsClient.WaitForEKSNodeGroupToBeDeleted(eksMetadata.Name, ngMetadata.Name, wait) + if err != nil { + logger.WithError(err).Errorf("failed to delete EKS NodeGroup %s", ngMetadata.Name) + errOccurred = true + return + } + + launchTemplateName := fmt.Sprintf("%s-%s", eksMetadata.Name, ngPrefix) + err = provisioner.awsClient.DeleteLaunchTemplate(launchTemplateName) + if err != nil { + logger.WithError(err).Errorf("failed to delete EKS LaunchTemplate %s", launchTemplateName) + errOccurred = true + return + } + + logger.Debugf("Successfully deleted EKS NodeGroup %s", ngMetadata.Name) + }(ng, meta) + } + + wg.Wait() + + if errOccurred { + return errors.New("failed to delete one of the nodegroups") + } + + for ng := range nodeGroups { + delete(eksMetadata.NodeGroups, ng) + } + + return nil +} + // ProvisionCluster provisions EKS cluster. func (provisioner *EKSProvisioner) ProvisionCluster(cluster *model.Cluster) error { logger := provisioner.logger.WithField("cluster", cluster.ID) diff --git a/internal/provisioner/kops_provisioner.go b/internal/provisioner/kops_provisioner.go index 41c612062..8ff283f78 100644 --- a/internal/provisioner/kops_provisioner.go +++ b/internal/provisioner/kops_provisioner.go @@ -447,18 +447,24 @@ func (provisioner *KopsProvisioner) CheckClusterCreated(cluster *model.Cluster) return true, nil } -func (provisioner *KopsProvisioner) CreateNodes(cluster *model.Cluster) error { +// CreateNodegroups is a noop for KopsProvisioner. +func (provisioner *KopsProvisioner) CreateNodegroups(cluster *model.Cluster) error { return nil } -// CheckNodesCreated is a noop for KopsProvisioner. -func (provisioner *KopsProvisioner) CheckNodesCreated(cluster *model.Cluster) (bool, error) { +// CheckNodegroupsCreated is a noop for KopsProvisioner. +func (provisioner *KopsProvisioner) CheckNodegroupsCreated(cluster *model.Cluster) (bool, error) { // TODO: this is currently not implemented for kops. // Entire waiting logic happens as part of cluster creation therefore we // just skip this step and report cluster as created. return true, nil } +// DeleteNodegroup is a noop for KopsProvisioner. +func (provisioner *KopsProvisioner) DeleteNodegroup(cluster *model.Cluster) error { + return nil +} + // ProvisionCluster installs all the baseline kubernetes resources needed for // managing installations. This can be called on an already-provisioned cluster // to re-provision with the newest version of the resources. diff --git a/internal/supervisor/cluster.go b/internal/supervisor/cluster.go index e92d64c55..5c958680a 100644 --- a/internal/supervisor/cluster.go +++ b/internal/supervisor/cluster.go @@ -33,8 +33,9 @@ type ClusterProvisioner interface { PrepareCluster(cluster *model.Cluster) bool CreateCluster(cluster *model.Cluster) error CheckClusterCreated(cluster *model.Cluster) (bool, error) - CreateNodes(cluster *model.Cluster) error - CheckNodesCreated(cluster *model.Cluster) (bool, error) + CreateNodegroups(cluster *model.Cluster) error + CheckNodegroupsCreated(cluster *model.Cluster) (bool, error) + DeleteNodegroup(cluster *model.Cluster) error ProvisionCluster(cluster *model.Cluster) error UpgradeCluster(cluster *model.Cluster) error ResizeCluster(cluster *model.Cluster) error @@ -177,6 +178,8 @@ func (s *ClusterSupervisor) transitionCluster(cluster *model.Cluster, logger log return s.resizeCluster(cluster, logger) case model.ClusterStateNodegroupsCreationRequested: return s.createNodegroups(cluster, logger) + case model.ClusterStateNodegroupDeletionRequested: + return s.deleteNodegroup(cluster, logger) case model.ClusterStateRefreshMetadata: return s.refreshClusterMetadata(cluster, logger) case model.ClusterStateDeletionRequested: @@ -248,13 +251,13 @@ func (s *ClusterSupervisor) resizeCluster(cluster *model.Cluster, logger log.Fie } func (s *ClusterSupervisor) createNodegroups(cluster *model.Cluster, logger log.FieldLogger) string { - err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).CreateNodes(cluster) + err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).CreateNodegroups(cluster) if err != nil { logger.WithError(err).Error("Failed to create nodegroups") return model.ClusterStateNodegroupsCreationFailed } - _, err = s.provisioner.GetClusterProvisioner(cluster.Provisioner).CheckNodesCreated(cluster) + _, err = s.provisioner.GetClusterProvisioner(cluster.Provisioner).CheckNodegroupsCreated(cluster) if err != nil { logger.WithError(err).Error("Failed to create nodegroups") return model.ClusterStateNodegroupsCreationFailed @@ -264,6 +267,17 @@ func (s *ClusterSupervisor) createNodegroups(cluster *model.Cluster, logger log. return s.refreshClusterMetadata(cluster, logger) } +func (s *ClusterSupervisor) deleteNodegroup(cluster *model.Cluster, logger log.FieldLogger) string { + err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).DeleteNodegroup(cluster) + if err != nil { + logger.WithError(err).Error("Failed to delete nodegroup") + return model.ClusterStateNodegroupDeletionFailed + } + + logger.Info("Finished deleting nodegroup") + return s.refreshClusterMetadata(cluster, logger) +} + func (s *ClusterSupervisor) refreshClusterMetadata(cluster *model.Cluster, logger log.FieldLogger) string { err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).RefreshClusterMetadata(cluster) @@ -313,7 +327,7 @@ func (s *ClusterSupervisor) checkClusterCreated(cluster *model.Cluster, logger l return model.ClusterStateCreationInProgress } - err = s.provisioner.GetClusterProvisioner(cluster.Provisioner).CreateNodes(cluster) + err = s.provisioner.GetClusterProvisioner(cluster.Provisioner).CreateNodegroups(cluster) if err != nil { logger.WithError(err).Error("Failed to create cluster nodes") return model.ClusterStateCreationFailed @@ -324,7 +338,7 @@ func (s *ClusterSupervisor) checkClusterCreated(cluster *model.Cluster, logger l func (s *ClusterSupervisor) checkNodesCreated(cluster *model.Cluster, logger log.FieldLogger) string { - ready, err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).CheckNodesCreated(cluster) + ready, err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).CheckNodegroupsCreated(cluster) if err != nil { logger.WithError(err).Error("Failed to check if node creation finished") return model.ClusterStateCreationFailed diff --git a/internal/supervisor/cluster_test.go b/internal/supervisor/cluster_test.go index 03148724a..97eaf073c 100644 --- a/internal/supervisor/cluster_test.go +++ b/internal/supervisor/cluster_test.go @@ -87,7 +87,11 @@ func (p *mockClusterProvisionerOption) GetClusterProvisioner(provisioner string) type mockClusterProvisioner struct{} -func (p *mockClusterProvisioner) CreateNodes(cluster *model.Cluster) error { +func (p *mockClusterProvisioner) DeleteNodegroup(cluster *model.Cluster) error { + return nil +} + +func (p *mockClusterProvisioner) CreateNodegroups(cluster *model.Cluster) error { return nil } @@ -107,7 +111,7 @@ func (p *mockClusterProvisioner) CheckClusterCreated(cluster *model.Cluster) (bo return true, nil } -func (p *mockClusterProvisioner) CheckNodesCreated(cluster *model.Cluster) (bool, error) { +func (p *mockClusterProvisioner) CheckNodegroupsCreated(cluster *model.Cluster) (bool, error) { return true, nil } diff --git a/model/client.go b/model/client.go index 9d8b534e9..ec11957b7 100644 --- a/model/client.go +++ b/model/client.go @@ -293,6 +293,22 @@ func (c *Client) CreateNodegroups(clusterID string, request *CreateNodegroupsReq } } +func (c *Client) DeleteNodegroup(clusterID string, nodegroup string) (*ClusterDTO, error) { + resp, err := c.doDelete(c.buildURL("/api/cluster/%s/nodegroups/%s", clusterID, nodegroup)) + if err != nil { + return nil, err + } + defer closeBody(resp) + + switch resp.StatusCode { + case http.StatusAccepted: + return DTOFromReader[ClusterDTO](resp.Body) + + default: + return nil, errors.Errorf("failed with status code %d", resp.StatusCode) + } +} + // DeleteCluster deletes the given cluster and all resources contained therein. func (c *Client) DeleteCluster(clusterID string) error { resp, err := c.doDelete(c.buildURL("/api/cluster/%s", clusterID)) diff --git a/model/cluster_states.go b/model/cluster_states.go index 108353dbb..0fab2ad62 100644 --- a/model/cluster_states.go +++ b/model/cluster_states.go @@ -36,6 +36,10 @@ const ( ClusterStateNodegroupsCreationRequested = "nodegroups-creation-requested" // ClusterStateNodegroupsCreationFailed is a cluster that failed to create nodegroups. ClusterStateNodegroupsCreationFailed = "nodegroups-creation-failed" + // ClusterStateNodegroupDeletionRequested is a cluster in the process of deleting nodegroup. + ClusterStateNodegroupDeletionRequested = "nodegroup-deletion-requested" + // ClusterStateNodegroupDeletionFailed is a cluster that failed to delete nodegroup. + ClusterStateNodegroupDeletionFailed = "nodegroup-deletion-failed" // ClusterStateDeletionRequested is a cluster in the process of being deleted. ClusterStateDeletionRequested = "deletion-requested" // ClusterStateDeletionFailed is a cluster that failed deletion. @@ -83,6 +87,7 @@ var AllClusterStatesPendingWork = []string{ ClusterStateUpgradeRequested, ClusterStateResizeRequested, ClusterStateNodegroupsCreationRequested, + ClusterStateNodegroupDeletionRequested, ClusterStateDeletionRequested, } @@ -146,6 +151,11 @@ var ( ClusterStateNodegroupsCreationRequested, ClusterStateNodegroupsCreationFailed, }, + ClusterStateNodegroupDeletionRequested: { + ClusterStateStable, + ClusterStateNodegroupDeletionRequested, + ClusterStateNodegroupDeletionFailed, + }, ClusterStateDeletionRequested: { ClusterStateStable, ClusterStateCreationRequested, diff --git a/model/eks_metadata.go b/model/eks_metadata.go index 7432690f0..a1f0bad9f 100644 --- a/model/eks_metadata.go +++ b/model/eks_metadata.go @@ -305,6 +305,15 @@ func (em *EKSMetadata) ValidateNodegroupsCreateRequest(nodegroups map[string]Nod return nil } +// ValidateNodegroupDeleteRequest ensures that the nodegroup to delete exists. +func (em *EKSMetadata) ValidateNodegroupDeleteRequest(nodegroup string) error { + if _, f := em.NodeGroups[nodegroup]; !f { + return errors.Errorf("nodegroup %s not found to delete", nodegroup) + } + + return nil +} + // ApplyNodegroupsCreateRequest applies the nodegroups to create to the // KopsMetadata. func (em *EKSMetadata) ApplyNodegroupsCreateRequest(request *CreateNodegroupsRequest) { @@ -340,6 +349,16 @@ func (em *EKSMetadata) ApplyNodegroupsCreateRequest(request *CreateNodegroupsReq } +// ApplyNodegroupDeleteRequest applies the nodegroup to delete to the +// KopsMetadata. +func (em *EKSMetadata) ApplyNodegroupDeleteRequest(nodegroup string) { + em.ChangeRequest = &EKSMetadataRequestedState{ + NodeGroups: map[string]NodeGroupMetadata{ + nodegroup: {}, + }, + } +} + func (em *EKSMetadata) GetCommonMetadata() ProvisionerMetadata { workerNodeGroup := em.NodeGroups[NodeGroupWorker] return ProvisionerMetadata{ From 320525a92a982d661ee8629656f032b0764dcf0c Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Tue, 25 Apr 2023 12:38:37 +0600 Subject: [PATCH 02/20] updated --- cmd/cloud/nodegroup.go | 2 +- internal/provisioner/eks_provisioner.go | 19 +++++-------------- model/client.go | 2 +- model/eks_metadata.go | 9 ++++++++- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/cmd/cloud/nodegroup.go b/cmd/cloud/nodegroup.go index ef1f22ce8..1db5468b0 100644 --- a/cmd/cloud/nodegroup.go +++ b/cmd/cloud/nodegroup.go @@ -118,7 +118,7 @@ func deleteNodegroup(flags clusterNodegroupDeleteFlags) error { cluster, err := client.DeleteNodegroup(flags.clusterID, flags.nodegroup) if err != nil { - return errors.Wrap(err, "failed to delete nodegroups") + return errors.Wrap(err, "failed to delete nodegroup") } if err = printJSON(cluster); err != nil { diff --git a/internal/provisioner/eks_provisioner.go b/internal/provisioner/eks_provisioner.go index 101bac8c1..2130c39a4 100644 --- a/internal/provisioner/eks_provisioner.go +++ b/internal/provisioner/eks_provisioner.go @@ -370,22 +370,13 @@ func (provisioner *EKSProvisioner) DeleteNodegroup(cluster *model.Cluster) error return errors.New("error: EKS metadata not set when deleting EKS NodeGroup") } - if eksMetadata.NodeGroups == nil || len(eksMetadata.NodeGroups) == 0 { - logger.Info("No EKS NodeGroup available to delete") - return nil - } - - nodeGroups := make(map[string]model.NodeGroupMetadata) - if eksMetadata.ChangeRequest != nil { - for ng := range eksMetadata.ChangeRequest.NodeGroups { - if metadata, found := eksMetadata.NodeGroups[ng]; found { - nodeGroups[ng] = metadata - } else { - logger.Warnf("EKS NodeGroup for %s not found in cluster metadata", ng) - } - } + changeRequest := eksMetadata.ChangeRequest + if changeRequest == nil || changeRequest.NodeGroups == nil { + return errors.New("error: EKS NodeGroup change request not set when deleting EKS NodeGroup") } + nodeGroups := changeRequest.NodeGroups + var wg sync.WaitGroup var errOccurred bool diff --git a/model/client.go b/model/client.go index ec11957b7..d848f28df 100644 --- a/model/client.go +++ b/model/client.go @@ -294,7 +294,7 @@ func (c *Client) CreateNodegroups(clusterID string, request *CreateNodegroupsReq } func (c *Client) DeleteNodegroup(clusterID string, nodegroup string) (*ClusterDTO, error) { - resp, err := c.doDelete(c.buildURL("/api/cluster/%s/nodegroups/%s", clusterID, nodegroup)) + resp, err := c.doDelete(c.buildURL("/api/cluster/%s/nodegroup/%s", clusterID, nodegroup)) if err != nil { return nil, err } diff --git a/model/eks_metadata.go b/model/eks_metadata.go index a1f0bad9f..de90d5126 100644 --- a/model/eks_metadata.go +++ b/model/eks_metadata.go @@ -352,9 +352,16 @@ func (em *EKSMetadata) ApplyNodegroupsCreateRequest(request *CreateNodegroupsReq // ApplyNodegroupDeleteRequest applies the nodegroup to delete to the // KopsMetadata. func (em *EKSMetadata) ApplyNodegroupDeleteRequest(nodegroup string) { + + if em.NodeGroups == nil { + return + } + em.ChangeRequest = &EKSMetadataRequestedState{ NodeGroups: map[string]NodeGroupMetadata{ - nodegroup: {}, + nodegroup: { + Name: em.NodeGroups[nodegroup].Name, + }, }, } } From 45d477bff537acdfa2799dc8b72bb9b9c23a8c7c Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Tue, 25 Apr 2023 20:19:01 +0600 Subject: [PATCH 03/20] Used plural name --- internal/provisioner/eks_provisioner.go | 2 +- internal/provisioner/kops_provisioner.go | 2 +- internal/supervisor/cluster.go | 4 ++-- internal/supervisor/cluster_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/provisioner/eks_provisioner.go b/internal/provisioner/eks_provisioner.go index 2130c39a4..7227707e5 100644 --- a/internal/provisioner/eks_provisioner.go +++ b/internal/provisioner/eks_provisioner.go @@ -362,7 +362,7 @@ func (provisioner *EKSProvisioner) CheckNodegroupsCreated(cluster *model.Cluster } // DeleteNodegroup deletes the EKS nodegroup. -func (provisioner *EKSProvisioner) DeleteNodegroup(cluster *model.Cluster) error { +func (provisioner *EKSProvisioner) DeleteNodegroups(cluster *model.Cluster) error { logger := provisioner.logger.WithField("cluster", cluster.ID) eksMetadata := cluster.ProvisionerMetadataEKS diff --git a/internal/provisioner/kops_provisioner.go b/internal/provisioner/kops_provisioner.go index 8ff283f78..ea1d94157 100644 --- a/internal/provisioner/kops_provisioner.go +++ b/internal/provisioner/kops_provisioner.go @@ -461,7 +461,7 @@ func (provisioner *KopsProvisioner) CheckNodegroupsCreated(cluster *model.Cluste } // DeleteNodegroup is a noop for KopsProvisioner. -func (provisioner *KopsProvisioner) DeleteNodegroup(cluster *model.Cluster) error { +func (provisioner *KopsProvisioner) DeleteNodegroups(cluster *model.Cluster) error { return nil } diff --git a/internal/supervisor/cluster.go b/internal/supervisor/cluster.go index 5c958680a..61337b777 100644 --- a/internal/supervisor/cluster.go +++ b/internal/supervisor/cluster.go @@ -35,7 +35,7 @@ type ClusterProvisioner interface { CheckClusterCreated(cluster *model.Cluster) (bool, error) CreateNodegroups(cluster *model.Cluster) error CheckNodegroupsCreated(cluster *model.Cluster) (bool, error) - DeleteNodegroup(cluster *model.Cluster) error + DeleteNodegroups(cluster *model.Cluster) error ProvisionCluster(cluster *model.Cluster) error UpgradeCluster(cluster *model.Cluster) error ResizeCluster(cluster *model.Cluster) error @@ -268,7 +268,7 @@ func (s *ClusterSupervisor) createNodegroups(cluster *model.Cluster, logger log. } func (s *ClusterSupervisor) deleteNodegroup(cluster *model.Cluster, logger log.FieldLogger) string { - err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).DeleteNodegroup(cluster) + err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).DeleteNodegroups(cluster) if err != nil { logger.WithError(err).Error("Failed to delete nodegroup") return model.ClusterStateNodegroupDeletionFailed diff --git a/internal/supervisor/cluster_test.go b/internal/supervisor/cluster_test.go index 97eaf073c..c82facae5 100644 --- a/internal/supervisor/cluster_test.go +++ b/internal/supervisor/cluster_test.go @@ -87,7 +87,7 @@ func (p *mockClusterProvisionerOption) GetClusterProvisioner(provisioner string) type mockClusterProvisioner struct{} -func (p *mockClusterProvisioner) DeleteNodegroup(cluster *model.Cluster) error { +func (p *mockClusterProvisioner) DeleteNodegroups(cluster *model.Cluster) error { return nil } From 28fe6502b7aa5a675ee696274f6b415afbfb13a2 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Tue, 25 Apr 2023 20:22:13 +0600 Subject: [PATCH 04/20] fixed comments --- internal/provisioner/eks_provisioner.go | 2 +- internal/provisioner/kops_provisioner.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/provisioner/eks_provisioner.go b/internal/provisioner/eks_provisioner.go index 7227707e5..5d7c91c8c 100644 --- a/internal/provisioner/eks_provisioner.go +++ b/internal/provisioner/eks_provisioner.go @@ -361,7 +361,7 @@ func (provisioner *EKSProvisioner) CheckNodegroupsCreated(cluster *model.Cluster return true, nil } -// DeleteNodegroup deletes the EKS nodegroup. +// DeleteNodegroups deletes the EKS nodegroup. func (provisioner *EKSProvisioner) DeleteNodegroups(cluster *model.Cluster) error { logger := provisioner.logger.WithField("cluster", cluster.ID) diff --git a/internal/provisioner/kops_provisioner.go b/internal/provisioner/kops_provisioner.go index ea1d94157..6adb415fd 100644 --- a/internal/provisioner/kops_provisioner.go +++ b/internal/provisioner/kops_provisioner.go @@ -460,7 +460,7 @@ func (provisioner *KopsProvisioner) CheckNodegroupsCreated(cluster *model.Cluste return true, nil } -// DeleteNodegroup is a noop for KopsProvisioner. +// DeleteNodegroups is a noop for KopsProvisioner. func (provisioner *KopsProvisioner) DeleteNodegroups(cluster *model.Cluster) error { return nil } From 7b6bad8e25b1c7ccd4acf69a8b07c28f8764daac Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Wed, 3 May 2023 13:02:37 +0600 Subject: [PATCH 05/20] Use VPC CNI as Networking --- internal/provisioner/eks_provisioner.go | 39 +++++-------------------- internal/tools/aws/eks.go | 15 ++++++++++ model/cluster_request.go | 2 ++ model/eks_metadata.go | 1 - 4 files changed, 24 insertions(+), 33 deletions(-) diff --git a/internal/provisioner/eks_provisioner.go b/internal/provisioner/eks_provisioner.go index 7599dba2c..b9b896460 100644 --- a/internal/provisioner/eks_provisioner.go +++ b/internal/provisioner/eks_provisioner.go @@ -5,7 +5,6 @@ package provisioner import ( - "context" "encoding/base64" "fmt" "os" @@ -21,7 +20,6 @@ import ( "github.com/mattermost/mattermost-cloud/model" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ) @@ -161,7 +159,6 @@ func (provisioner *EKSProvisioner) CheckClusterCreated(cluster *model.Cluster) ( } eksMetadata.ClusterRoleARN = *eksCluster.RoleArn - eksMetadata.Networking = model.NetworkingCalico eksMetadata.VPC = changeRequest.VPC eksMetadata.Version = changeRequest.Version @@ -170,27 +167,6 @@ func (provisioner *EKSProvisioner) CheckClusterCreated(cluster *model.Cluster) ( return false, errors.Wrap(err, "failed to store cluster") } - // To install Calico Networking, We need to delete VPC CNI plugin (aws-node) - // and install Calico CNI plugin before creating any pods - k8sClient, err := provisioner.getKubeClient(cluster) - if err != nil { - return false, errors.Wrap(err, "failed to initialize K8s client from kube config") - } - - // Delete aws-node daemonset to disable VPC CNI plugin - _ = k8sClient.Clientset.AppsV1().DaemonSets("kube-system").Delete(context.Background(), "aws-node", metav1.DeleteOptions{}) - - var files []k8s.ManifestFile - files = append(files, k8s.ManifestFile{ - Path: "manifests/eks/calico-eks.yaml", - DeployNamespace: "kube-system", - }) - - err = k8sClient.CreateFromFiles(files) - if err != nil { - return false, err - } - return true, nil } @@ -349,11 +325,17 @@ func (provisioner *EKSProvisioner) CheckNodesCreated(cluster *model.Cluster) (bo return false, errors.New("one of the EKS NodeGroups failed to become active") } + err := provisioner.awsClient.InstallEKSAddons(cluster) + if err != nil { + return false, errors.Wrap(err, "failed to install EKS EBS Addon") + } + eksMetadata.NodeRoleARN = changeRequest.NodeRoleARN eksMetadata.AMI = changeRequest.AMI eksMetadata.MaxPodsPerNode = changeRequest.MaxPodsPerNode + eksMetadata.Networking = model.NetworkingVpcCni - err := provisioner.clusterUpdateStore.UpdateCluster(cluster) + err = provisioner.clusterUpdateStore.UpdateCluster(cluster) if err != nil { return false, errors.Wrap(err, "failed to store cluster") } @@ -370,13 +352,6 @@ func (provisioner *EKSProvisioner) ProvisionCluster(cluster *model.Cluster) erro return errors.New("expected EKS metadata not to be nil when using EKS Provisioner") } - // TODO: ideally we would do it as part of cluster creation as this - // also is async operation. - err := provisioner.awsClient.InstallEKSAddons(cluster) - if err != nil { - return errors.Wrap(err, "failed to install EKS EBS Addon") - } - kubeConfigPath, err := provisioner.getKubeConfigPath(cluster) if err != nil { return errors.Wrap(err, "failed to get kubeconfig file path") diff --git a/internal/tools/aws/eks.go b/internal/tools/aws/eks.go index b5b1f11df..05fc52af9 100644 --- a/internal/tools/aws/eks.go +++ b/internal/tools/aws/eks.go @@ -126,6 +126,21 @@ func (a *Client) InstallEKSAddons(cluster *model.Cluster) error { return errors.Wrap(err, "failed to create ebs-csi addon") } + input = eks.CreateAddonInput{ + AddonName: aws.String("vpc-cni"), + ClusterName: aws.String(cluster.ProvisionerMetadataEKS.Name), + ConfigurationValues: aws.String("{\"env\":{\"ENABLE_PREFIX_DELEGATION\":\"true\"}}"), + ResolveConflicts: eksTypes.ResolveConflictsOverwrite, + } + _, err = a.Service().eks.CreateAddon(context.TODO(), &input) + if err != nil { + // In case addon already configured we do not want to fail. + if IsErrorResourceInUseException(err) { + return nil + } + return errors.Wrap(err, "failed to create vpc-cni addon") + } + return nil } diff --git a/model/cluster_request.go b/model/cluster_request.go index 9bca80ce0..efd562903 100644 --- a/model/cluster_request.go +++ b/model/cluster_request.go @@ -17,6 +17,8 @@ const ( NetworkingCalico = "calico" // NetworkingAmazon is Amazon networking plugin. NetworkingAmazon = "amazon-vpc-routed-eni" + // NetworkingVpcCni is Amazon VPC CNI networking plugin. + NetworkingVpcCni = "vpc-cni" ) // CreateClusterRequest specifies the parameters for a new cluster. diff --git a/model/eks_metadata.go b/model/eks_metadata.go index 7432690f0..734f956cd 100644 --- a/model/eks_metadata.go +++ b/model/eks_metadata.go @@ -82,7 +82,6 @@ func (em *EKSMetadata) ApplyClusterCreateRequest(createRequest *CreateClusterReq Version: createRequest.Version, AMI: createRequest.AMI, MaxPodsPerNode: createRequest.MaxPodsPerNode, - Networking: createRequest.Networking, VPC: createRequest.VPC, ClusterRoleARN: createRequest.ClusterRoleARN, NodeRoleARN: createRequest.NodeRoleARN, From 3df308b09edd1a39e61dbf03db4b5daa565264c1 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Wed, 3 May 2023 15:17:59 +0600 Subject: [PATCH 06/20] removed calico cni manifest --- manifests/eks/calico-eks.yaml | 4785 --------------------------------- 1 file changed, 4785 deletions(-) delete mode 100644 manifests/eks/calico-eks.yaml diff --git a/manifests/eks/calico-eks.yaml b/manifests/eks/calico-eks.yaml deleted file mode 100644 index 6358100b4..000000000 --- a/manifests/eks/calico-eks.yaml +++ /dev/null @@ -1,4785 +0,0 @@ ---- -# Source: calico/templates/calico-kube-controllers.yaml -# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers ---- -# Source: calico/templates/calico-kube-controllers.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - namespace: kube-system ---- -# Source: calico/templates/calico-node.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system ---- -# Source: calico/templates/calico-config.yaml -# This ConfigMap is used to configure a self-hosted Calico installation. -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-config - namespace: kube-system -data: - # Typha is disabled. - typha_service_name: "none" - # Configure the backend to use. - calico_backend: "vxlan" - - # Configure the MTU to use for workload interfaces and tunnels. - # By default, MTU is auto-detected, and explicitly setting this field should not be required. - # You can override auto-detection by providing a non-zero value. - veth_mtu: "0" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "log_file_path": "/var/log/calico/cni/cni.log", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "calico-ipam" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - }, - { - "type": "bandwidth", - "capabilities": {"bandwidth": true} - } - ] - } ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: bgpconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPConfiguration - listKind: BGPConfigurationList - plural: bgpconfigurations - singular: bgpconfiguration - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: BGPConfiguration contains the configuration for any BGP routing. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPConfigurationSpec contains the values of the BGP configuration. - properties: - asNumber: - description: 'ASNumber is the default AS number used by a node. [Default: - 64512]' - format: int32 - type: integer - bindMode: - description: BindMode indicates whether to listen for BGP connections - on all addresses (None) or only on the node's canonical IP address - Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen - for BGP connections on all addresses. - type: string - communities: - description: Communities is a list of BGP community values and their - arbitrary names for tagging routes. - items: - description: Community contains standard or large community value - and its name. - properties: - name: - description: Name given to community value. - type: string - value: - description: Value must be of format `aa:nn` or `aa:nn:mm`. - For standard community use `aa:nn` format, where `aa` and - `nn` are 16 bit number. For large community use `aa:nn:mm` - format, where `aa`, `nn` and `mm` are 32 bit number. Where, - `aa` is an AS Number, `nn` and `mm` are per-AS identifier. - pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ - type: string - type: object - type: array - ignoredInterfaces: - description: IgnoredInterfaces indicates the network interfaces that - needs to be excluded when reading device routes. - items: - type: string - type: array - listenPort: - description: ListenPort is the port where BGP protocol should listen. - Defaults to 179 - maximum: 65535 - minimum: 1 - type: integer - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: INFO]' - type: string - nodeMeshMaxRestartTime: - description: Time to allow for software restart for node-to-mesh peerings. When - specified, this is configured as the graceful restart timeout. When - not specified, the BIRD default of 120s is used. This field can - only be set on the default BGPConfiguration instance and requires - that NodeMesh is enabled - type: string - nodeMeshPassword: - description: Optional BGP password for full node-to-mesh peerings. - This field can only be set on the default BGPConfiguration instance - and requires that NodeMesh is enabled - properties: - secretKeyRef: - description: Selects a key of a secret in the node pod's namespace. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - type: object - nodeToNodeMeshEnabled: - description: 'NodeToNodeMeshEnabled sets whether full node to node - BGP mesh is enabled. [Default: true]' - type: boolean - prefixAdvertisements: - description: PrefixAdvertisements contains per-prefix advertisement - configuration. - items: - description: PrefixAdvertisement configures advertisement properties - for the specified CIDR. - properties: - cidr: - description: CIDR for which properties should be advertised. - type: string - communities: - description: Communities can be list of either community names - already defined in `Specs.Communities` or community value - of format `aa:nn` or `aa:nn:mm`. For standard community use - `aa:nn` format, where `aa` and `nn` are 16 bit number. For - large community use `aa:nn:mm` format, where `aa`, `nn` and - `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and - `mm` are per-AS identifier. - items: - type: string - type: array - type: object - type: array - serviceClusterIPs: - description: ServiceClusterIPs are the CIDR blocks from which service - cluster IPs are allocated. If specified, Calico will advertise these - blocks, as well as any cluster IPs within them. - items: - description: ServiceClusterIPBlock represents a single allowed ClusterIP - CIDR block. - properties: - cidr: - type: string - type: object - type: array - serviceExternalIPs: - description: ServiceExternalIPs are the CIDR blocks for Kubernetes - Service External IPs. Kubernetes Service ExternalIPs will only be - advertised if they are within one of these blocks. - items: - description: ServiceExternalIPBlock represents a single allowed - External IP CIDR block. - properties: - cidr: - type: string - type: object - type: array - serviceLoadBalancerIPs: - description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes - Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress - IPs will only be advertised if they are within one of these blocks. - items: - description: ServiceLoadBalancerIPBlock represents a single allowed - LoadBalancer IP CIDR block. - properties: - cidr: - type: string - type: object - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: bgppeers.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPPeer - listKind: BGPPeerList - plural: bgppeers - singular: bgppeer - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPPeerSpec contains the specification for a BGPPeer resource. - properties: - asNumber: - description: The AS Number of the peer. - format: int32 - type: integer - keepOriginalNextHop: - description: Option to keep the original nexthop field when routes - are sent to a BGP Peer. Setting "true" configures the selected BGP - Peers node to use the "next hop keep;" instead of "next hop self;"(default) - in the specific branch of the Node on "bird.cfg". - type: boolean - maxRestartTime: - description: Time to allow for software restart. When specified, - this is configured as the graceful restart timeout. When not specified, - the BIRD default of 120s is used. - type: string - node: - description: The node name identifying the Calico node instance that - is targeted by this peer. If this is not set, and no nodeSelector - is specified, then this BGP peer selects all nodes in the cluster. - type: string - nodeSelector: - description: Selector for the nodes that should have this peering. When - this is set, the Node field must be empty. - type: string - numAllowedLocalASNumbers: - description: Maximum number of local AS numbers that are allowed in - the AS path for received routes. This removes BGP loop prevention - and should only be used if absolutely necesssary. - format: int32 - type: integer - password: - description: Optional BGP password for the peerings generated by this - BGPPeer resource. - properties: - secretKeyRef: - description: Selects a key of a secret in the node pod's namespace. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - type: object - peerIP: - description: The IP address of the peer followed by an optional port - number to peer with. If port number is given, format should be `[]:port` - or `:` for IPv4. If optional port number is not set, - and this peer IP and ASNumber belongs to a calico/node with ListenPort - set in BGPConfiguration, then we use that port to peer. - type: string - peerSelector: - description: Selector for the remote nodes to peer with. When this - is set, the PeerIP and ASNumber fields must be empty. For each - peering between the local node and selected remote nodes, we configure - an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, - and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The - remote AS number comes from the remote node's NodeBGPSpec.ASNumber, - or the global default if that is not set. - type: string - reachableBy: - description: Add an exact, i.e. /32, static route toward peer IP in - order to prevent route flapping. ReachableBy contains the address - of the gateway which peer can be reached by. - type: string - sourceAddress: - description: Specifies whether and how to configure a source address - for the peerings generated by this BGPPeer resource. Default value - "UseNodeIP" means to configure the node IP as the source address. "None" - means not to configure a source address. - type: string - ttlSecurity: - description: TTLSecurity enables the generalized TTL security mechanism - (GTSM) which protects against spoofed packets by ignoring received - packets with a smaller than expected TTL value. The provided value - is the number of hops (edges) between the peers. - type: integer - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: blockaffinities.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BlockAffinity - listKind: BlockAffinityList - plural: blockaffinities - singular: blockaffinity - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BlockAffinitySpec contains the specification for a BlockAffinity - resource. - properties: - cidr: - type: string - deleted: - description: Deleted indicates that this block affinity is being deleted. - This field is a string for compatibility with older releases that - mistakenly treat this field as a string. - type: string - node: - type: string - state: - type: string - required: - - cidr - - deleted - - node - - state - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: caliconodestatuses.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: CalicoNodeStatus - listKind: CalicoNodeStatusList - plural: caliconodestatuses - singular: caliconodestatus - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus - resource. - properties: - classes: - description: Classes declares the types of information to monitor - for this calico/node, and allows for selective status reporting - about certain subsets of information. - items: - type: string - type: array - node: - description: The node name identifies the Calico node instance for - node status. - type: string - updatePeriodSeconds: - description: UpdatePeriodSeconds is the period at which CalicoNodeStatus - should be updated. Set to 0 to disable CalicoNodeStatus refresh. - Maximum update period is one day. - format: int32 - type: integer - type: object - status: - description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. - No validation needed for status since it is updated by Calico. - properties: - agent: - description: Agent holds agent status on the node. - properties: - birdV4: - description: BIRDV4 represents the latest observed status of bird4. - properties: - lastBootTime: - description: LastBootTime holds the value of lastBootTime - from bird.ctl output. - type: string - lastReconfigurationTime: - description: LastReconfigurationTime holds the value of lastReconfigTime - from bird.ctl output. - type: string - routerID: - description: Router ID used by bird. - type: string - state: - description: The state of the BGP Daemon. - type: string - version: - description: Version of the BGP daemon - type: string - type: object - birdV6: - description: BIRDV6 represents the latest observed status of bird6. - properties: - lastBootTime: - description: LastBootTime holds the value of lastBootTime - from bird.ctl output. - type: string - lastReconfigurationTime: - description: LastReconfigurationTime holds the value of lastReconfigTime - from bird.ctl output. - type: string - routerID: - description: Router ID used by bird. - type: string - state: - description: The state of the BGP Daemon. - type: string - version: - description: Version of the BGP daemon - type: string - type: object - type: object - bgp: - description: BGP holds node BGP status. - properties: - numberEstablishedV4: - description: The total number of IPv4 established bgp sessions. - type: integer - numberEstablishedV6: - description: The total number of IPv6 established bgp sessions. - type: integer - numberNotEstablishedV4: - description: The total number of IPv4 non-established bgp sessions. - type: integer - numberNotEstablishedV6: - description: The total number of IPv6 non-established bgp sessions. - type: integer - peersV4: - description: PeersV4 represents IPv4 BGP peers status on the node. - items: - description: CalicoNodePeer contains the status of BGP peers - on the node. - properties: - peerIP: - description: IP address of the peer whose condition we are - reporting. - type: string - since: - description: Since the state or reason last changed. - type: string - state: - description: State is the BGP session state. - type: string - type: - description: Type indicates whether this peer is configured - via the node-to-node mesh, or via en explicit global or - per-node BGPPeer object. - type: string - type: object - type: array - peersV6: - description: PeersV6 represents IPv6 BGP peers status on the node. - items: - description: CalicoNodePeer contains the status of BGP peers - on the node. - properties: - peerIP: - description: IP address of the peer whose condition we are - reporting. - type: string - since: - description: Since the state or reason last changed. - type: string - state: - description: State is the BGP session state. - type: string - type: - description: Type indicates whether this peer is configured - via the node-to-node mesh, or via en explicit global or - per-node BGPPeer object. - type: string - type: object - type: array - required: - - numberEstablishedV4 - - numberEstablishedV6 - - numberNotEstablishedV4 - - numberNotEstablishedV6 - type: object - lastUpdated: - description: LastUpdated is a timestamp representing the server time - when CalicoNodeStatus object last updated. It is represented in - RFC3339 form and is in UTC. - format: date-time - nullable: true - type: string - routes: - description: Routes reports routes known to the Calico BGP daemon - on the node. - properties: - routesV4: - description: RoutesV4 represents IPv4 routes on the node. - items: - description: CalicoNodeRoute contains the status of BGP routes - on the node. - properties: - destination: - description: Destination of the route. - type: string - gateway: - description: Gateway for the destination. - type: string - interface: - description: Interface for the destination - type: string - learnedFrom: - description: LearnedFrom contains information regarding - where this route originated. - properties: - peerIP: - description: If sourceType is NodeMesh or BGPPeer, IP - address of the router that sent us this route. - type: string - sourceType: - description: Type of the source where a route is learned - from. - type: string - type: object - type: - description: Type indicates if the route is being used for - forwarding or not. - type: string - type: object - type: array - routesV6: - description: RoutesV6 represents IPv6 routes on the node. - items: - description: CalicoNodeRoute contains the status of BGP routes - on the node. - properties: - destination: - description: Destination of the route. - type: string - gateway: - description: Gateway for the destination. - type: string - interface: - description: Interface for the destination - type: string - learnedFrom: - description: LearnedFrom contains information regarding - where this route originated. - properties: - peerIP: - description: If sourceType is NodeMesh or BGPPeer, IP - address of the router that sent us this route. - type: string - sourceType: - description: Type of the source where a route is learned - from. - type: string - type: object - type: - description: Type indicates if the route is being used for - forwarding or not. - type: string - type: object - type: array - type: object - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clusterinformations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: ClusterInformation - listKind: ClusterInformationList - plural: clusterinformations - singular: clusterinformation - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: ClusterInformation contains the cluster specific information. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ClusterInformationSpec contains the values of describing - the cluster. - properties: - calicoVersion: - description: CalicoVersion is the version of Calico that the cluster - is running - type: string - clusterGUID: - description: ClusterGUID is the GUID of the cluster - type: string - clusterType: - description: ClusterType describes the type of the cluster - type: string - datastoreReady: - description: DatastoreReady is used during significant datastore migrations - to signal to components such as Felix that it should wait before - accessing the datastore. - type: boolean - variant: - description: Variant declares which variant of Calico should be active. - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: felixconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: FelixConfiguration - listKind: FelixConfigurationList - plural: felixconfigurations - singular: felixconfiguration - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: Felix Configuration contains the configuration for Felix. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: FelixConfigurationSpec contains the values of the Felix configuration. - properties: - allowIPIPPacketsFromWorkloads: - description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads - [Default: false]' - type: boolean - allowVXLANPacketsFromWorkloads: - description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads - [Default: false]' - type: boolean - awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' - enum: - - DoNothing - - Enable - - Disable - type: string - bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. [Default: - true]' - type: boolean - bpfDataIfacePattern: - description: BPFDataIfacePattern is a regular expression that controls - which interfaces Felix should attach BPF programs to in order to - catch traffic to/from the network. This needs to match the interfaces - that Calico workload traffic flows over as well as any interfaces - that handle incoming traffic to nodeports and services from outside - the cluster. It should not match the workload interfaces (usually - named cali...). - type: string - bpfDisableUnprivileged: - description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled - sysctl to disable unprivileged use of BPF. This ensures that unprivileged - users cannot access Calico''s BPF maps and cannot insert their own - BPF programs to interfere with Calico''s. [Default: true]' - type: boolean - bpfEnabled: - description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. - [Default: false]' - type: boolean - bpfEnforceRPF: - description: 'BPFEnforceRPF enforce strict RPF on all host interfaces - with BPF programs regardless of what is the per-interfaces or global - setting. Possible values are Disabled, Strict or Loose. [Default: - Strict]' - type: string - bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit - mark that is set on connections from an external client to a local - service. This mark allows us to control how packets of that connection - are routed within the host and how is routing interpreted by RPF - check. [Default: 0]' - type: integer - bpfExternalServiceMode: - description: 'BPFExternalServiceMode in BPF mode, controls how connections - from outside the cluster to services (node ports and cluster IPs) - are forwarded to remote workloads. If set to "Tunnel" then both - request and response traffic is tunneled to the remote node. If - set to "DSR", the request traffic is tunneled but the response traffic - is sent directly from the remote node. In "DSR" mode, the remote - node appears to use the IP of the ingress node; this requires a - permissive L2 network. [Default: Tunnel]' - type: string - bpfHostConntrackBypass: - description: 'BPFHostConntrackBypass Controls whether to bypass Linux - conntrack in BPF mode for workloads and services. [Default: true - - bypass Linux conntrack]' - type: boolean - bpfKubeProxyEndpointSlicesEnabled: - description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls - whether Felix's embedded kube-proxy accepts EndpointSlices or not. - type: boolean - bpfKubeProxyIptablesCleanupEnabled: - description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF - mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s - iptables chains. Should only be enabled if kube-proxy is not running. [Default: - true]' - type: boolean - bpfKubeProxyMinSyncPeriod: - description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the - minimum time between updates to the dataplane for Felix''s embedded - kube-proxy. Lower values give reduced set-up latency. Higher values - reduce Felix CPU usage by batching up more work. [Default: 1s]' - type: string - bpfL3IfacePattern: - description: BPFL3IfacePattern is a regular expression that allows - to list tunnel devices like wireguard or vxlan (i.e., L3 devices) - in addition to BPFDataIfacePattern. That is, tunnel interfaces not - created by Calico, that Calico workload traffic flows over as well - as any interfaces that handle incoming traffic to nodeports and - services from outside the cluster. - type: string - bpfLogLevel: - description: 'BPFLogLevel controls the log level of the BPF programs - when in BPF dataplane mode. One of "Off", "Info", or "Debug". The - logs are emitted to the BPF trace pipe, accessible with the command - `tc exec bpf debug`. [Default: Off].' - type: string - bpfMapSizeConntrack: - description: 'BPFMapSizeConntrack sets the size for the conntrack - map. This map must be large enough to hold an entry for each active - connection. Warning: changing the size of the conntrack map can - cause disruption.' - type: integer - bpfMapSizeIPSets: - description: BPFMapSizeIPSets sets the size for ipsets map. The IP - sets map must be large enough to hold an entry for each endpoint - matched by every selector in the source/destination matches in network - policy. Selectors such as "all()" can result in large numbers of - entries (one entry per endpoint in that case). - type: integer - bpfMapSizeIfState: - description: BPFMapSizeIfState sets the size for ifstate map. The - ifstate map must be large enough to hold an entry for each device - (host + workloads) on a host. - type: integer - bpfMapSizeNATAffinity: - type: integer - bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. - This is the total number of endpoints. This is mostly more than - the size of the number of services. - type: integer - bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end - map. FrontendMap should be large enough to hold an entry for each - nodeport, external IP and each port in each service. - type: integer - bpfMapSizeRoute: - description: BPFMapSizeRoute sets the size for the routes map. The - routes map should be large enough to hold one entry per workload - and a handful of entries per host (enough to cover its own IPs and - tunnel IPs). - type: integer - bpfPSNATPorts: - anyOf: - - type: integer - - type: string - description: 'BPFPSNATPorts sets the range from which we randomly - pick a port if there is a source port collision. This should be - within the ephemeral range as defined by RFC 6056 (1024–65535) and - preferably outside the ephemeral ranges used by common operating - systems. Linux uses 32768–60999, while others mostly use the IANA - defined range 49152–65535. It is not necessarily a problem if this - range overlaps with the operating systems. Both ends of the range - are inclusive. [Default: 20000:29999]' - pattern: ^.* - x-kubernetes-int-or-string: true - bpfPolicyDebugEnabled: - description: BPFPolicyDebugEnabled when true, Felix records detailed - information about the BPF policy programs, which can be examined - with the calico-bpf command-line tool. - type: boolean - chainInsertMode: - description: 'ChainInsertMode controls whether Felix hooks the kernel''s - top-level iptables chains by inserting a rule at the top of the - chain or by appending a rule at the bottom. insert is the safe default - since it prevents Calico''s rules from being bypassed. If you switch - to append mode, be sure that the other rules in the chains signal - acceptance by falling through to the Calico rules, otherwise the - Calico policy will be bypassed. [Default: insert]' - type: string - dataplaneDriver: - description: DataplaneDriver filename of the external dataplane driver - to use. Only used if UseInternalDataplaneDriver is set to false. - type: string - dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." - type: string - debugDisableLogDropping: - type: boolean - debugMemoryProfilePath: - type: string - debugSimulateCalcGraphHangAfter: - type: string - debugSimulateDataplaneHangAfter: - type: string - defaultEndpointToHostAction: - description: 'DefaultEndpointToHostAction controls what happens to - traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' - type: string - deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. - type: integer - deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. - type: string - deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. - type: string - disableConntrackInvalidCheck: - type: boolean - endpointReportingDelay: - type: string - endpointReportingEnabled: - type: boolean - externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. - items: - type: string - type: array - failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports - and CIDRs that Felix will allow incoming traffic to host endpoints - on irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. For - back-compatibility, if the protocol is not specified, it defaults - to "tcp". If a CIDR is not specified, it will allow traffic from - all addresses. To disable all inbound host ports, use the value - none. The default value allows ssh access and DHCP. [Default: tcp:22, - udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' - items: - description: ProtoPort is combination of protocol, port, and CIDR. - Protocol and port must be specified. - properties: - net: - type: string - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports - and CIDRs that Felix will allow outgoing traffic from host endpoints - to irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. For - back-compatibility, if the protocol is not specified, it defaults - to "tcp". If a CIDR is not specified, it will allow traffic from - all addresses. To disable all outbound host ports, use the value - none. The default value opens etcd''s standard ports to ensure that - Felix does not get cut off from etcd as well as allowing DHCP and - DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, - tcp:6667, udp:53, udp:67]' - items: - description: ProtoPort is combination of protocol, port, and CIDR. - Protocol and port must be specified. - properties: - net: - type: string - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - featureDetectOverride: - description: FeatureDetectOverride is used to override feature detection - based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. - type: string - featureGates: - description: FeatureGates is used to enable or disable tech-preview - Calico features. Values are specified in a comma separated list - with no spaces, example; "BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false". - This is used to enable features that are not fully production ready. - type: string - floatingIPs: - description: FloatingIPs configures whether or not Felix will program - non-OpenStack floating IP addresses. (OpenStack-derived floating - IPs are always programmed, regardless of this setting.) - enum: - - Enabled - - Disabled - type: string - genericXDPEnabled: - description: 'GenericXDPEnabled enables Generic XDP so network cards - that don''t support XDP offload or driver modes can use XDP. This - is not recommended since it doesn''t provide better performance - than iptables. [Default: false]' - type: boolean - healthEnabled: - type: boolean - healthHost: - type: string - healthPort: - type: integer - healthTimeoutOverrides: - description: HealthTimeoutOverrides allows the internal watchdog timeouts - of individual subcomponents to be overriden. This is useful for - working around "false positive" liveness timeouts that can occur - in particularly stressful workloads or if CPU is constrained. For - a list of active subcomponents, see Felix's logs. - items: - properties: - name: - type: string - timeout: - type: string - required: - - name - - timeout - type: object - type: array - interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' - type: string - interfacePrefix: - description: 'InterfacePrefix is the interface name prefix that identifies - workload endpoints and so distinguishes them from host endpoint - interfaces. Note: in environments other than bare metal, the orchestrators - configure this appropriately. For example our Kubernetes and Docker - integrations set the ''cali'' value, and our OpenStack integration - sets the ''tap'' value. [Default: cali]' - type: string - interfaceRefreshInterval: - description: InterfaceRefreshInterval is the period at which Felix - rescans local interfaces to verify their state. The rescan can be - disabled by setting the interval to 0. - type: string - ipipEnabled: - description: 'IPIPEnabled overrides whether Felix should configure - an IPIP interface on the host. Optional as Felix determines this - based on the existing IP pools. [Default: nil (unset)]' - type: boolean - ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' - type: string - iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - type: string - iptablesFilterAllowAction: - type: string - iptablesLockFilePath: - description: 'IptablesLockFilePath is the location of the iptables - lock file. You may need to change this if the lock file is not in - its standard location (for example if you have mapped it into Felix''s - container at a different path). [Default: /run/xtables.lock]' - type: string - iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' - type: string - iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' - type: string - iptablesMangleAllowAction: - type: string - iptablesMarkMask: - description: 'IptablesMarkMask is the mask that Felix selects its - IPTables Mark bits from. Should be a 32 bit hexadecimal number with - at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' - format: int32 - type: integer - iptablesNATOutgoingInterfaceFilter: - type: string - iptablesPostWriteCheckInterval: - description: 'IptablesPostWriteCheckInterval is the period after Felix - has done a write to the dataplane that it schedules an extra read - back in order to check the write was not clobbered by another process. - This should only occur if another application on the system doesn''t - respect the iptables lock. [Default: 1s]' - type: string - iptablesRefreshInterval: - description: 'IptablesRefreshInterval is the period at which Felix - re-checks the IP sets in the dataplane to ensure that no other process - has accidentally broken Calico''s rules. Set to 0 to disable IP - sets refresh. Note: the default for this value is lower than the - other refresh intervals as a workaround for a Linux kernel bug that - was fixed in kernel version 4.11. If you are using v4.11 or greater - you may want to set this to, a higher value to reduce Felix CPU - usage. [Default: 10s]' - type: string - ipv6Support: - description: IPv6Support controls whether Felix enables support for - IPv6 (if supported by the in-use dataplane). - type: boolean - kubeNodePortRanges: - description: 'KubeNodePortRanges holds list of port ranges used for - service node ports. Only used if felix detects kube-proxy running - in ipvs mode. Felix uses these ranges to separate host and workload - traffic. [Default: 30000:32767].' - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - logDebugFilenameRegex: - description: LogDebugFilenameRegex controls which source code files - have their Debug log output included in the logs. Only logs from - files with names that match the given regular expression are included. The - filter only applies to Debug level logs. - type: string - logFilePath: - description: 'LogFilePath is the full path to the Felix log. Set to - none to disable file logging. [Default: /var/log/calico/felix.log]' - type: string - logPrefix: - description: 'LogPrefix is the log prefix that Felix uses when rendering - LOG rules. [Default: calico-packet]' - type: string - logSeverityFile: - description: 'LogSeverityFile is the log severity above which logs - are sent to the log file. [Default: Info]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - logSeveritySys: - description: 'LogSeveritySys is the log severity above which logs - are sent to the syslog. Set to None for no logging to syslog. [Default: - Info]' - type: string - maxIpsetSize: - type: integer - metadataAddr: - description: 'MetadataAddr is the IP address or domain name of the - server that can answer VM queries for cloud-init metadata. In OpenStack, - this corresponds to the machine running nova-api (or in Ubuntu, - nova-api-metadata). A value of none (case insensitive) means that - Felix should not set up any NAT rule for the metadata path. [Default: - 127.0.0.1]' - type: string - metadataPort: - description: 'MetadataPort is the port of the metadata server. This, - combined with global.MetadataAddr (if not ''None''), is used to - set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. - In most cases this should not need to be changed [Default: 8775].' - type: integer - mtuIfacePattern: - description: MTUIfacePattern is a regular expression that controls - which interfaces Felix should scan in order to calculate the host's - MTU. This should not match workload interfaces (usually named cali...). - type: string - natOutgoingAddress: - description: NATOutgoingAddress specifies an address to use when performing - source NAT for traffic in a natOutgoing pool that is leaving the - network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) - type: string - natPortRange: - anyOf: - - type: integer - - type: string - description: NATPortRange specifies the range of ports that is used - for port mapping when doing outgoing NAT. When unset the default - behavior of the network stack is used. - pattern: ^.* - x-kubernetes-int-or-string: true - netlinkTimeout: - type: string - openstackRegion: - description: 'OpenstackRegion is the name of the region that a particular - Felix belongs to. In a multi-region Calico/OpenStack deployment, - this must be configured somehow for each Felix (here in the datamodel, - or in felix.cfg or the environment on each compute node), and must - match the [calico] openstack_region value configured in neutron.conf - on each node. [Default: Empty]' - type: string - policySyncPathPrefix: - description: 'PolicySyncPathPrefix is used to by Felix to communicate - policy changes to external services, like Application layer policy. - [Default: Empty]' - type: string - prometheusGoMetricsEnabled: - description: 'PrometheusGoMetricsEnabled disables Go runtime metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusMetricsEnabled: - description: 'PrometheusMetricsEnabled enables the Prometheus metrics - server in Felix if set to true. [Default: false]' - type: boolean - prometheusMetricsHost: - description: 'PrometheusMetricsHost is the host that the Prometheus - metrics server should bind to. [Default: empty]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. [Default: 9091]' - type: integer - prometheusProcessMetricsEnabled: - description: 'PrometheusProcessMetricsEnabled disables process metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusWireGuardMetricsEnabled: - description: 'PrometheusWireGuardMetricsEnabled disables wireguard - metrics collection, which the Prometheus client does by default, - when set to false. This reduces the number of metrics reported, - reducing Prometheus load. [Default: true]' - type: boolean - removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. - type: boolean - reportingInterval: - description: 'ReportingInterval is the interval at which Felix reports - its status into the datastore or 0 to disable. Must be non-zero - in OpenStack deployments. [Default: 30s]' - type: string - reportingTTL: - description: 'ReportingTTL is the time-to-live setting for process-wide - status reports. [Default: 90s]' - type: string - routeRefreshInterval: - description: 'RouteRefreshInterval is the period at which Felix re-checks - the routes in the dataplane to ensure that no other process has - accidentally broken Calico''s rules. Set to 0 to disable route refresh. - [Default: 90s]' - type: string - routeSource: - description: 'RouteSource configures where Felix gets its routing - information. - WorkloadIPs: use workload endpoints to construct - routes. - CalicoIPAM: the default - use IPAM data to construct routes.' - type: string - routeSyncDisabled: - description: RouteSyncDisabled will disable all operations performed - on the route table. Set to true to run in network-policy mode only. - type: boolean - routeTableRange: - description: Deprecated in favor of RouteTableRanges. Calico programs - additional Linux route tables for various purposes. RouteTableRange - specifies the indices of the route tables that Calico should use. - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - routeTableRanges: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRanges specifies a set of table index ranges - that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. - items: - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - type: array - serviceLoopPrevention: - description: 'When service IP advertisement is enabled, prevent routing - loops to service IPs that are not in use, by dropping or rejecting - packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", - in which case such routing loops continue to be allowed. [Default: - Drop]' - type: string - sidecarAccelerationEnabled: - description: 'SidecarAccelerationEnabled enables experimental sidecar - acceleration [Default: false]' - type: boolean - usageReportingEnabled: - description: 'UsageReportingEnabled reports anonymous Calico version - number and cluster size to projectcalico.org. Logs warnings returned - by the usage server. For example, if a significant security vulnerability - has been discovered in the version of Calico being used. [Default: - true]' - type: boolean - usageReportingInitialDelay: - description: 'UsageReportingInitialDelay controls the minimum delay - before Felix makes a report. [Default: 300s]' - type: string - usageReportingInterval: - description: 'UsageReportingInterval controls the interval at which - Felix makes reports. [Default: 86400s]' - type: string - useInternalDataplaneDriver: - description: UseInternalDataplaneDriver, if true, Felix will use its - internal dataplane programming logic. If false, it will launch - an external dataplane driver and communicate with it over protobuf. - type: boolean - vxlanEnabled: - description: 'VXLANEnabled overrides whether Felix should create the - VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix - determines this based on the existing IP pools. [Default: nil (unset)]' - type: boolean - vxlanMTU: - description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' - type: integer - vxlanMTUV6: - description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' - type: integer - vxlanPort: - type: integer - vxlanVNI: - type: integer - wireguardEnabled: - description: 'WireguardEnabled controls whether Wireguard is enabled - for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). - [Default: false]' - type: boolean - wireguardEnabledV6: - description: 'WireguardEnabledV6 controls whether Wireguard is enabled - for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). - [Default: false]' - type: boolean - wireguardHostEncryptionEnabled: - description: 'WireguardHostEncryptionEnabled controls whether Wireguard - host-to-host encryption is enabled. [Default: false]' - type: boolean - wireguardInterfaceName: - description: 'WireguardInterfaceName specifies the name to use for - the IPv4 Wireguard interface. [Default: wireguard.cali]' - type: string - wireguardInterfaceNameV6: - description: 'WireguardInterfaceNameV6 specifies the name to use for - the IPv6 Wireguard interface. [Default: wg-v6.cali]' - type: string - wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive - option. Set 0 to disable. [Default: 0]' - type: string - wireguardListeningPort: - description: 'WireguardListeningPort controls the listening port used - by IPv4 Wireguard. [Default: 51820]' - type: integer - wireguardListeningPortV6: - description: 'WireguardListeningPortV6 controls the listening port - used by IPv6 Wireguard. [Default: 51821]' - type: integer - wireguardMTU: - description: 'WireguardMTU controls the MTU on the IPv4 Wireguard - interface. See Configuring MTU [Default: 1440]' - type: integer - wireguardMTUV6: - description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard - interface. See Configuring MTU [Default: 1420]' - type: integer - wireguardRoutingRulePriority: - description: 'WireguardRoutingRulePriority controls the priority value - to use for the Wireguard routing rule. [Default: 99]' - type: integer - workloadSourceSpoofing: - description: WorkloadSourceSpoofing controls whether pods can use - the allowedSourcePrefixes annotation to send traffic with a source - IP address that is not theirs. This is disabled by default. When - set to "Any", pods can request any prefix. - type: string - xdpEnabled: - description: 'XDPEnabled enables XDP acceleration for suitable untracked - incoming deny rules. [Default: true]' - type: boolean - xdpRefreshInterval: - description: 'XDPRefreshInterval is the period at which Felix re-checks - all XDP state to ensure that no other process has accidentally broken - Calico''s BPF maps or attached programs. Set to 0 to disable XDP - refresh. [Default: 90s]' - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: globalnetworkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkPolicy - listKind: GlobalNetworkPolicyList - plural: globalnetworkpolicies - singular: globalnetworkpolicy - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - applyOnForward: - description: ApplyOnForward indicates to apply the rules in this policy - on forward traffic. - type: boolean - doNotTrack: - description: DoNotTrack indicates whether packets matched by the rules - in this policy should go through the data plane's connection tracking, - such as Linux conntrack. If True, the rules in this policy are - applied before any data plane connection tracking, and packets allowed - by this policy are marked as not to be tracked. - type: boolean - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, `global()` - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, `global()` - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, `global()` - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, `global()` - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - namespaceSelector: - description: NamespaceSelector is an optional field for an expression - used to select a pod based on namespaces. - type: string - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - preDNAT: - description: PreDNAT indicates to apply the rules in this policy before - any DNAT. - type: boolean - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress rules are present in the policy. The - default is: \n - [ PolicyTypeIngress ], if there are no Egress rules - (including the case where there are also no Ingress rules) \n - - [ PolicyTypeEgress ], if there are Egress rules but no Ingress - rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are - both Ingress and Egress rules. \n When the policy is read back again, - Types will always be one of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: globalnetworksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkSet - listKind: GlobalNetworkSetList - plural: globalnetworksets - singular: globalnetworkset - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs - that share labels to allow rules to refer to them via selectors. The labels - of GlobalNetworkSet are not namespaced. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: GlobalNetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: hostendpoints.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: HostEndpoint - listKind: HostEndpointList - plural: hostendpoints - singular: hostendpoint - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HostEndpointSpec contains the specification for a HostEndpoint - resource. - properties: - expectedIPs: - description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. - If \"InterfaceName\" is not present, Calico will look for an interface - matching any of the IPs in the list and apply policy to that. Note: - \tWhen using the selector match criteria in an ingress or egress - security Policy \tor Profile, Calico converts the selector into - a set of IP addresses. For host \tendpoints, the ExpectedIPs field - is used for that purpose. (If only the interface \tname is specified, - Calico does not learn the IPs of the interface for use in match - \tcriteria.)" - items: - type: string - type: array - interfaceName: - description: "Either \"*\", or the name of a specific Linux interface - to apply policy to; or empty. \"*\" indicates that this HostEndpoint - governs all traffic to, from or through the default network namespace - of the host named by the \"Node\" field; entering and leaving that - namespace via any interface, including those from/to non-host-networked - local workloads. \n If InterfaceName is not \"*\", this HostEndpoint - only governs traffic that enters or leaves the host through the - specific interface named by InterfaceName, or - when InterfaceName - is empty - through the specific interface that has one of the IPs - in ExpectedIPs. Therefore, when InterfaceName is empty, at least - one expected IP must be specified. Only external interfaces (such - as \"eth0\") are supported here; it isn't possible for a HostEndpoint - to protect traffic through a specific local workload interface. - \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; - initially just pre-DNAT policy. Please check Calico documentation - for the latest position." - type: string - node: - description: The node name identifying the Calico node instance. - type: string - ports: - description: Ports contains the endpoint's named ports, which may - be referenced in security policy rules. - items: - properties: - name: - type: string - port: - type: integer - protocol: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - required: - - name - - port - - protocol - type: object - type: array - profiles: - description: A list of identifiers of security Profile objects that - apply to this endpoint. Each profile is applied in the order that - they appear in this list. Profile rules are applied after the selector-based - security policy. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamblocks.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMBlock - listKind: IPAMBlockList - plural: ipamblocks - singular: ipamblock - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMBlockSpec contains the specification for an IPAMBlock - resource. - properties: - affinity: - description: Affinity of the block, if this block has one. If set, - it will be of the form "host:". If not set, this block - is not affine to a host. - type: string - allocations: - description: Array of allocations in-use within this block. nil entries - mean the allocation is free. For non-nil entries at index i, the - index is the ordinal of the allocation within this block and the - value is the index of the associated attributes in the Attributes - array. - items: - type: integer - # TODO: This nullable is manually added in. We should update controller-gen - # to handle []*int properly itself. - nullable: true - type: array - attributes: - description: Attributes is an array of arbitrary metadata associated - with allocations in the block. To find attributes for a given allocation, - use the value of the allocation's entry in the Allocations array - as the index of the element in this array. - items: - properties: - handle_id: - type: string - secondary: - additionalProperties: - type: string - type: object - type: object - type: array - cidr: - description: The block's CIDR. - type: string - deleted: - description: Deleted is an internal boolean used to workaround a limitation - in the Kubernetes API whereby deletion will not return a conflict - error if the block has been updated. It should not be set manually. - type: boolean - sequenceNumber: - default: 0 - description: We store a sequence number that is updated each time - the block is written. Each allocation will also store the sequence - number of the block at the time of its creation. When releasing - an IP, passing the sequence number associated with the allocation - allows us to protect against a race condition and ensure the IP - hasn't been released and re-allocated since the release request. - format: int64 - type: integer - sequenceNumberForAllocation: - additionalProperties: - format: int64 - type: integer - description: Map of allocated ordinal within the block to sequence - number of the block at the time of allocation. Kubernetes does not - allow numerical keys for maps, so the key is cast to a string. - type: object - strictAffinity: - description: StrictAffinity on the IPAMBlock is deprecated and no - longer used by the code. Use IPAMConfig StrictAffinity instead. - type: boolean - unallocated: - description: Unallocated is an ordered list of allocations which are - free in the block. - items: - type: integer - type: array - required: - - allocations - - attributes - - cidr - - strictAffinity - - unallocated - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamconfigs.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMConfig - listKind: IPAMConfigList - plural: ipamconfigs - singular: ipamconfig - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMConfigSpec contains the specification for an IPAMConfig - resource. - properties: - autoAllocateBlocks: - type: boolean - maxBlocksPerHost: - description: MaxBlocksPerHost, if non-zero, is the max number of blocks - that can be affine to each host. - maximum: 2147483647 - minimum: 0 - type: integer - strictAffinity: - type: boolean - required: - - autoAllocateBlocks - - strictAffinity - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamhandles.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMHandle - listKind: IPAMHandleList - plural: ipamhandles - singular: ipamhandle - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMHandleSpec contains the specification for an IPAMHandle - resource. - properties: - block: - additionalProperties: - type: integer - type: object - deleted: - type: boolean - handleID: - type: string - required: - - block - - handleID - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ippools.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPPool - listKind: IPPoolList - plural: ippools - singular: ippool - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPPoolSpec contains the specification for an IPPool resource. - properties: - allowedUses: - description: AllowedUse controls what the IP pool will be used for. If - not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility - items: - type: string - type: array - blockSize: - description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 122 for IPv6. - type: integer - cidr: - description: The pool CIDR. - type: string - disableBGPExport: - description: 'Disable exporting routes from this IP Pool''s CIDR over - BGP. [Default: false]' - type: boolean - disabled: - description: When disabled is true, Calico IPAM will not assign addresses - from this pool. - type: boolean - ipip: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - properties: - enabled: - description: When enabled is true, ipip tunneling will be used - to deliver packets to destinations within this pool. - type: boolean - mode: - description: The IPIP mode. This can be one of "always" or "cross-subnet". A - mode of "always" will also use IPIP tunneling for routing to - destination IP addresses within this pool. A mode of "cross-subnet" - will only use IPIP tunneling when the destination node is on - a different subnet to the originating node. The default value - (if not specified) is "always". - type: string - type: object - ipipMode: - description: Contains configuration for IPIP tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling - is disabled). - type: string - nat-outgoing: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - type: boolean - natOutgoing: - description: When natOutgoing is true, packets sent from Calico networked - containers in this pool to destinations outside of this pool will - be masqueraded. - type: boolean - nodeSelector: - description: Allows IPPool to allocate for a specific node by label - selector. - type: string - vxlanMode: - description: Contains configuration for VXLAN tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. VXLAN - tunneling is disabled). - type: string - required: - - cidr - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipreservations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPReservation - listKind: IPReservationList - plural: ipreservations - singular: ipreservation - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPReservationSpec contains the specification for an IPReservation - resource. - properties: - reservedCIDRs: - description: ReservedCIDRs is a list of CIDRs and/or IP addresses - that Calico IPAM will exclude from new allocations. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kubecontrollersconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: KubeControllersConfiguration - listKind: KubeControllersConfigurationList - plural: kubecontrollersconfigurations - singular: kubecontrollersconfiguration - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KubeControllersConfigurationSpec contains the values of the - Kubernetes controllers configuration. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host endpoints. - Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation of - host endpoints for every node. [Default: Disabled]' - type: string - type: object - leakGracePeriod: - description: 'LeakGracePeriod is the period used by the controller - to determine if an IP address has been leaked. Set to 0 - to disable IP garbage collection. [Default: 15m]' - type: string - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - type: object - debugProfilePort: - description: DebugProfilePort configures the port to serve memory - and cpu profiles on. If not specified, profiling is disabled. - format: int32 - type: integer - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. Set to 0 to disable. [Default: 9094]' - type: integer - required: - - controllers - type: object - status: - description: KubeControllersConfigurationStatus represents the status - of the configuration. It's useful for admins to be able to see the actual - config that was applied, which can be modified by environment variables - on the kube-controllers process. - properties: - environmentVars: - additionalProperties: - type: string - description: EnvironmentVars contains the environment variables on - the kube-controllers that influenced the RunningConfig. - type: object - runningConfig: - description: RunningConfig contains the effective config that is running - in the kube-controllers pod, after merging the API resource with - any environment variables. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace - controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host - endpoints. Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation - of host endpoints for every node. [Default: Disabled]' - type: string - type: object - leakGracePeriod: - description: 'LeakGracePeriod is the period used by the - controller to determine if an IP address has been leaked. - Set to 0 to disable IP garbage collection. [Default: - 15m]' - type: string - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - type: object - debugProfilePort: - description: DebugProfilePort configures the port to serve memory - and cpu profiles on. If not specified, profiling is disabled. - format: int32 - type: integer - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which - logs are sent to the stdout. [Default: Info]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. Set to 0 to disable. [Default: - 9094]' - type: integer - required: - - controllers - type: object - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: networkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkPolicy - listKind: NetworkPolicyList - plural: networkpolicies - singular: networkpolicy - preserveUnknownFields: false - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, `global()` - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, `global()` - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, `global()` - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, `global()` - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress are present in the policy. The default - is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including - the case where there are also no Ingress rules) \n - [ PolicyTypeEgress - ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, - PolicyTypeEgress ], if there are both Ingress and Egress rules. - \n When the policy is read back again, Types will always be one - of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: networksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkSet - listKind: NetworkSetList - plural: networksets - singular: networkset - preserveUnknownFields: false - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/calico-kube-controllers-rbac.yaml -# Include a clusterrole for the kube-controllers component, -# and bind it to the calico-kube-controllers serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -rules: - # Nodes are watched to monitor for deletions. - - apiGroups: [""] - resources: - - nodes - verbs: - - watch - - list - - get - # Pods are watched to check for existence as part of IPAM controller. - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipreservations - verbs: - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - - watch - # Pools are watched to maintain a mapping of blocks to IP pools. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - verbs: - - list - - watch - # kube-controllers manages hostendpoints. - - apiGroups: ["crd.projectcalico.org"] - resources: - - hostendpoints - verbs: - - get - - list - - create - - update - - delete - # Needs access to update clusterinformations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - clusterinformations - verbs: - - get - - list - - create - - update - - watch - # KubeControllersConfiguration is where it gets its config - - apiGroups: ["crd.projectcalico.org"] - resources: - - kubecontrollersconfigurations - verbs: - # read its own config - - get - # create a default if none exists - - create - # update status - - update - # watch for changes - - watch ---- -# Source: calico/templates/calico-node-rbac.yaml -# Include a clusterrole for the calico-node DaemonSet, -# and bind it to the calico-node serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-node -rules: - # Used for creating service account tokens to be used by the CNI plugin - - apiGroups: [""] - resources: - - serviceaccounts/token - resourceNames: - - calico-node - verbs: - - create - # The CNI plugin needs to get pods, nodes, and namespaces. - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - # EndpointSlices are used for Service-based network policy rule - # enforcement. - - apiGroups: ["discovery.k8s.io"] - resources: - - endpointslices - verbs: - - watch - - list - - apiGroups: [""] - resources: - - endpoints - - services - verbs: - # Used to discover service IPs for advertisement. - - watch - - list - # Used to discover Typhas. - - get - # Pod CIDR auto-detection on kubeadm needs access to config maps. - - apiGroups: [""] - resources: - - configmaps - verbs: - - get - - apiGroups: [""] - resources: - - nodes/status - verbs: - # Needed for clearing NodeNetworkUnavailable flag. - - patch - # Calico stores some configuration information in node annotations. - - update - # Watch for changes to Kubernetes NetworkPolicies. - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - # Used by Calico for policy information. - - apiGroups: [""] - resources: - - pods - - namespaces - - serviceaccounts - verbs: - - list - - watch - # The CNI plugin patches pods/status. - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - # Calico monitors various CRDs for config. - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - ipreservations - - ipamblocks - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - networksets - - clusterinformations - - hostendpoints - - blockaffinities - - caliconodestatuses - verbs: - - get - - list - - watch - # Calico must create and update some CRDs on startup. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - - felixconfigurations - - clusterinformations - verbs: - - create - - update - # Calico must update some CRDs. - - apiGroups: [ "crd.projectcalico.org" ] - resources: - - caliconodestatuses - verbs: - - update - # Calico stores some configuration information on the node. - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - watch - # These permissions are only required for upgrade from v2.6, and can - # be removed after upgrade or on fresh installations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - bgpconfigurations - - bgppeers - verbs: - - create - - update - # These permissions are required for Calico CNI to perform IPAM allocations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - # The CNI plugin and calico/node need to be able to create a default - # IPAMConfiguration - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipamconfigs - verbs: - - get - - create - # Block affinities must also be watchable by confd for route aggregation. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - verbs: - - watch - # The Calico IPAM migration needs to get daemonsets. These permissions can be - # removed if not upgrading from an installation using host-local IPAM. - - apiGroups: ["apps"] - resources: - - daemonsets - verbs: - - get ---- -# Source: calico/templates/calico-kube-controllers-rbac.yaml -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system ---- -# Source: calico/templates/calico-node-rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system ---- -# Source: calico/templates/calico-node.yaml -# This manifest installs the calico-node container, as well -# as the CNI plugins and network config on -# each master and worker node in a Kubernetes cluster. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node -spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Make sure calico-node gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: calico-node - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: - # This container performs upgrade from host-local IPAM to calico-ipam. - # It can be deleted if this is a fresh installation, or if you have already - # upgraded to use calico-ipam. - - name: upgrade-ipam - image: docker.io/calico/cni:v3.25.0 - imagePullPolicy: IfNotPresent - command: ["/opt/cni/bin/calico-ipam", "-upgrade"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - - name: FELIX_AWSSRCDSTCHECK - value: Disable - volumeMounts: - - mountPath: /var/lib/cni/networks - name: host-local-net-dir - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - securityContext: - privileged: true - # This container installs the CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: docker.io/calico/cni:v3.25.0 - imagePullPolicy: IfNotPresent - command: ["/opt/cni/bin/install"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Prevents the container from sleeping forever. - - name: SLEEP - value: "false" - - name: FELIX_AWSSRCDSTCHECK - value: Disable - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - securityContext: - privileged: true - # This init container mounts the necessary filesystems needed by the BPF data plane - # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed - # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. - - name: "mount-bpffs" - image: docker.io/calico/node:v3.25.0 - imagePullPolicy: IfNotPresent - command: ["calico-node", "-init", "-best-effort"] - env: - - name: FELIX_AWSSRCDSTCHECK - value: Disable - volumeMounts: - - mountPath: /sys/fs - name: sys-fs - # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host - # so that it outlives the init container. - mountPropagation: Bidirectional - - mountPath: /var/run/calico - name: var-run-calico - # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host - # so that it outlives the init container. - mountPropagation: Bidirectional - # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, - # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. - - mountPath: /nodeproc - name: nodeproc - readOnly: true - securityContext: - privileged: true - containers: - # Runs calico-node container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - - name: calico-node - image: docker.io/calico/node:v3.25.0 - imagePullPolicy: IfNotPresent - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Choose the backend to use. - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,bgp" - # Auto-detect the BGP IP address. - - name: IP - value: "autodetect" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "Never" - # Enable or Disable VXLAN on the default IP pool. - - name: CALICO_IPV4POOL_VXLAN - value: "CrossSubnet" - # Enable or Disable VXLAN on the default IPv6 IP pool. - - name: CALICO_IPV6POOL_VXLAN - value: "CrossSubnet" - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the VXLAN tunnel device. - - name: FELIX_VXLANMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the Wireguard tunnel device. - - name: FELIX_WIREGUARDMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. This should fall within `--cluster-cidr`. - # - name: CALICO_IPV4POOL_CIDR - # value: "192.168.0.0/16" - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_AWSSRCDSTCHECK - value: Disable - securityContext: - privileged: true - resources: - requests: - cpu: 250m - lifecycle: - preStop: - exec: - command: - - /bin/calico-node - - -shutdown - livenessProbe: - exec: - command: - - /bin/calico-node - - -felix-live - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - timeoutSeconds: 10 - readinessProbe: - exec: - command: - - /bin/calico-node - - -felix-ready - periodSeconds: 10 - timeoutSeconds: 10 - volumeMounts: - # For maintaining CNI plugin API credentials. - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - readOnly: false - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - - name: policysync - mountPath: /var/run/nodeagent - # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the - # parent directory. - - name: bpffs - mountPath: /sys/fs/bpf - - name: cni-log-dir - mountPath: /var/log/calico/cni - readOnly: true - volumes: - # Used by calico-node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: sys-fs - hostPath: - path: /sys/fs/ - type: DirectoryOrCreate - - name: bpffs - hostPath: - path: /sys/fs/bpf - type: Directory - # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. - - name: nodeproc - hostPath: - path: /proc - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - # Used to access CNI logs. - - name: cni-log-dir - hostPath: - path: /var/log/calico/cni - # Mount in the directory for host-local IPAM allocations. This is - # used when upgrading from host-local to calico-ipam, and can be removed - # if not using the upgrade-ipam init container. - - name: host-local-net-dir - hostPath: - path: /var/lib/cni/networks - # Used to create per-pod Unix Domain Sockets - - name: policysync - hostPath: - type: DirectoryOrCreate - path: /var/run/nodeagent ---- -# Source: calico/templates/calico-kube-controllers.yaml -# See https://github.com/projectcalico/kube-controllers -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - # The controllers can only have a single active instance. - replicas: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: - nodeSelector: - kubernetes.io/os: linux - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - serviceAccountName: calico-kube-controllers - priorityClassName: system-cluster-critical - containers: - - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.25.0 - imagePullPolicy: IfNotPresent - env: - # Choose which controllers to run. - - name: ENABLED_CONTROLLERS - value: node - - name: DATASTORE_TYPE - value: kubernetes - livenessProbe: - exec: - command: - - /usr/bin/check-status - - -l - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - timeoutSeconds: 10 - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r - periodSeconds: 10 From 5782aeda27443ee52152dd56a8026705f126729c Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Thu, 4 May 2023 15:50:30 +0600 Subject: [PATCH 07/20] Remove -local from cluster name --- internal/provisioner/eks_provisioner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/provisioner/eks_provisioner.go b/internal/provisioner/eks_provisioner.go index 7599dba2c..448fb3151 100644 --- a/internal/provisioner/eks_provisioner.go +++ b/internal/provisioner/eks_provisioner.go @@ -68,7 +68,7 @@ func (provisioner *EKSProvisioner) PrepareCluster(cluster *model.Cluster) bool { } // Generate the EKS name using the cluster id. - cluster.ProvisionerMetadataEKS.Name = fmt.Sprintf("%s-eks-k8s-local", cluster.ID) + cluster.ProvisionerMetadataEKS.Name = fmt.Sprintf("%s-eks-k8s", cluster.ID) return true } From 2c6fb72cdd79c5c7e0ef0ab2b0c80e7ede3bcf54 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Thu, 4 May 2023 17:39:26 +0600 Subject: [PATCH 08/20] Set InstanceType in LaunchTemplate --- internal/provisioner/eks_provisioner.go | 20 ++++++++++++++------ internal/tools/aws/ec2.go | 2 ++ internal/tools/aws/eks.go | 1 - model/aws_types.go | 1 + 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/internal/provisioner/eks_provisioner.go b/internal/provisioner/eks_provisioner.go index 448fb3151..6924066dc 100644 --- a/internal/provisioner/eks_provisioner.go +++ b/internal/provisioner/eks_provisioner.go @@ -194,14 +194,14 @@ func (provisioner *EKSProvisioner) CheckClusterCreated(cluster *model.Cluster) ( return true, nil } -func (provisioner *EKSProvisioner) prepareLaunchTemplate(cluster *model.Cluster, ngPrefix string, withSG bool, logger log.FieldLogger) error { +func (provisioner *EKSProvisioner) prepareLaunchTemplate(cluster *model.Cluster, ngPrefix string, ngMetadata model.NodeGroupMetadata, logger log.FieldLogger) error { eksMetadata := cluster.ProvisionerMetadataEKS changeRequest := eksMetadata.ChangeRequest var err error var securityGroups []string - if withSG { + if ngMetadata.WithSecurityGroup { vpc := changeRequest.VPC if vpc == "" { vpc = eksMetadata.VPC @@ -218,6 +218,7 @@ func (provisioner *EKSProvisioner) prepareLaunchTemplate(cluster *model.Cluster, SecurityGroups: securityGroups, AMI: changeRequest.AMI, MaxPodsPerNode: changeRequest.MaxPodsPerNode, + InstanceType: ngMetadata.InstanceType, } if launchTemplateData.AMI == "" { @@ -270,7 +271,7 @@ func (provisioner *EKSProvisioner) CreateNodes(cluster *model.Cluster) error { defer wg.Done() logger.Debugf("Creating EKS NodeGroup %s", ngMetadata.Name) - err := provisioner.prepareLaunchTemplate(cluster, ngPrefix, ngMetadata.WithSecurityGroup, logger) + err := provisioner.prepareLaunchTemplate(cluster, ngPrefix, ngMetadata, logger) if err != nil { logger.WithError(err).Error("failed to ensure launch template") errOccurred = true @@ -332,7 +333,7 @@ func (provisioner *EKSProvisioner) CheckNodesCreated(cluster *model.Cluster) (bo eksMetadata.NodeGroups[ngPrefix] = model.NodeGroupMetadata{ Name: ngMetadata.Name, Type: ngMetadata.Type, - InstanceType: nodeGroup.InstanceTypes[0], + InstanceType: ngMetadata.InstanceType, MinCount: int64(ptr.ToInt32(nodeGroup.ScalingConfig.MinSize)), MaxCount: int64(ptr.ToInt32(nodeGroup.ScalingConfig.MaxSize)), WithPublicSubnet: ngMetadata.WithPublicSubnet, @@ -423,7 +424,7 @@ func (provisioner *EKSProvisioner) UpgradeCluster(cluster *model.Cluster) error ngMetadata.CopyMissingFieldsFrom(oldMetadata) changeRequest.NodeGroups[ngPrefix] = ngMetadata - err = provisioner.prepareLaunchTemplate(cluster, ngPrefix, ngMetadata.WithSecurityGroup, logger) + err = provisioner.prepareLaunchTemplate(cluster, ngPrefix, ngMetadata, logger) if err != nil { logger.WithError(err).Error("failed to ensure launch template") errOccurred = true @@ -570,6 +571,13 @@ func (provisioner *EKSProvisioner) ResizeCluster(cluster *model.Cluster) error { return } + err = provisioner.prepareLaunchTemplate(cluster, ngPrefix, ngMetadata, logger) + if err != nil { + logger.WithError(err).Error("failed to ensure launch template") + errOccurred = true + return + } + err = provisioner.awsClient.EnsureEKSNodeGroupMigrated(cluster, ngPrefix) if err != nil { logger.WithError(err).Errorf("failed to migrate EKS NodeGroup for %s", ngPrefix) @@ -592,7 +600,7 @@ func (provisioner *EKSProvisioner) ResizeCluster(cluster *model.Cluster) error { oldNodeGroup := eksMetadata.NodeGroups[ngPrefix] oldNodeGroup.Name = ngMetadata.Name - oldNodeGroup.InstanceType = nodeGroup.InstanceTypes[0] + oldNodeGroup.InstanceType = ngMetadata.InstanceType oldNodeGroup.MinCount = int64(ptr.ToInt32(nodeGroup.ScalingConfig.MinSize)) oldNodeGroup.MaxCount = int64(ptr.ToInt32(nodeGroup.ScalingConfig.MaxSize)) eksMetadata.NodeGroups[ngPrefix] = oldNodeGroup diff --git a/internal/tools/aws/ec2.go b/internal/tools/aws/ec2.go index b7b16be34..0bdcfe90b 100644 --- a/internal/tools/aws/ec2.go +++ b/internal/tools/aws/ec2.go @@ -190,6 +190,7 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { ImageId: aws.String(data.AMI), UserData: aws.String(encodedUserData), SecurityGroupIds: data.SecurityGroups, + InstanceType: ec2Types.InstanceType(data.InstanceType), }, LaunchTemplateName: aws.String(data.Name), }) @@ -226,6 +227,7 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { ImageId: aws.String(data.AMI), UserData: aws.String(encodedUserData), SecurityGroupIds: data.SecurityGroups, + InstanceType: ec2Types.InstanceType(data.InstanceType), }, LaunchTemplateName: aws.String(data.Name), }) diff --git a/internal/tools/aws/eks.go b/internal/tools/aws/eks.go index b5b1f11df..e9a9fa7bd 100644 --- a/internal/tools/aws/eks.go +++ b/internal/tools/aws/eks.go @@ -214,7 +214,6 @@ func (a *Client) createEKSNodeGroup(cluster *model.Cluster, ngPrefix string) (*e nodeGroupReq := eks.CreateNodegroupInput{ ClusterName: aws.String(clusterName), - InstanceTypes: []string{ngChangeRequest.InstanceType}, NodeRole: &changeRequest.NodeRoleARN, NodegroupName: aws.String(ngChangeRequest.Name), AmiType: eksTypes.AMITypesCustom, diff --git a/model/aws_types.go b/model/aws_types.go index 96724daeb..5d520603e 100644 --- a/model/aws_types.go +++ b/model/aws_types.go @@ -15,4 +15,5 @@ type LaunchTemplateData struct { AMI string MaxPodsPerNode int64 SecurityGroups []string + InstanceType string } From 5d8c6c71b941e82a3e3e83e4b17c634a92c56d80 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Thu, 4 May 2023 19:40:08 +0600 Subject: [PATCH 09/20] Associate public IP address by LaunchTemplate --- internal/provisioner/eks_provisioner.go | 17 +++++++---- internal/tools/aws/ec2.go | 38 +++++++++++++++++++------ model/aws_types.go | 13 +++++---- 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/internal/provisioner/eks_provisioner.go b/internal/provisioner/eks_provisioner.go index 6924066dc..b6fd87883 100644 --- a/internal/provisioner/eks_provisioner.go +++ b/internal/provisioner/eks_provisioner.go @@ -213,12 +213,13 @@ func (provisioner *EKSProvisioner) prepareLaunchTemplate(cluster *model.Cluster, } launchTemplateData := &model.LaunchTemplateData{ - Name: fmt.Sprintf("%s-%s", eksMetadata.Name, ngPrefix), - ClusterName: eksMetadata.Name, - SecurityGroups: securityGroups, - AMI: changeRequest.AMI, - MaxPodsPerNode: changeRequest.MaxPodsPerNode, - InstanceType: ngMetadata.InstanceType, + Name: fmt.Sprintf("%s-%s", eksMetadata.Name, ngPrefix), + ClusterName: eksMetadata.Name, + SecurityGroups: securityGroups, + AMI: changeRequest.AMI, + MaxPodsPerNode: changeRequest.MaxPodsPerNode, + InstanceType: ngMetadata.InstanceType, + WithPublicSubnet: ngMetadata.WithPublicSubnet, } if launchTemplateData.AMI == "" { @@ -271,6 +272,10 @@ func (provisioner *EKSProvisioner) CreateNodes(cluster *model.Cluster) error { defer wg.Done() logger.Debugf("Creating EKS NodeGroup %s", ngMetadata.Name) + oldMetadata := eksMetadata.NodeGroups[ngPrefix] + ngMetadata.CopyMissingFieldsFrom(oldMetadata) + changeRequest.NodeGroups[ngPrefix] = ngMetadata + err := provisioner.prepareLaunchTemplate(cluster, ngPrefix, ngMetadata, logger) if err != nil { logger.WithError(err).Error("failed to ensure launch template") diff --git a/internal/tools/aws/ec2.go b/internal/tools/aws/ec2.go index 0bdcfe90b..a5d958885 100644 --- a/internal/tools/aws/ec2.go +++ b/internal/tools/aws/ec2.go @@ -185,12 +185,23 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { userData := getLaunchTemplateUserData(eksCluster, data) encodedUserData := base64.StdEncoding.EncodeToString([]byte(userData)) + var networkInterfaces []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest + if data.WithPublicSubnet { + networkInterfaces = []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + { + DeviceIndex: aws.Int32(0), + AssociatePublicIpAddress: aws.Bool(true), + }, + } + } + launchTemplate, err := a.Service().ec2.CreateLaunchTemplate(context.TODO(), &ec2.CreateLaunchTemplateInput{ LaunchTemplateData: &ec2Types.RequestLaunchTemplateData{ - ImageId: aws.String(data.AMI), - UserData: aws.String(encodedUserData), - SecurityGroupIds: data.SecurityGroups, - InstanceType: ec2Types.InstanceType(data.InstanceType), + ImageId: aws.String(data.AMI), + UserData: aws.String(encodedUserData), + SecurityGroupIds: data.SecurityGroups, + InstanceType: ec2Types.InstanceType(data.InstanceType), + NetworkInterfaces: networkInterfaces, }, LaunchTemplateName: aws.String(data.Name), }) @@ -222,12 +233,23 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { userData := getLaunchTemplateUserData(eksCluster, data) encodedUserData := base64.StdEncoding.EncodeToString([]byte(userData)) + var networkInterfaces []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest + if data.WithPublicSubnet { + networkInterfaces = []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + { + DeviceIndex: aws.Int32(0), + AssociatePublicIpAddress: aws.Bool(true), + }, + } + } + launchTemplate, err := a.Service().ec2.CreateLaunchTemplateVersion(context.TODO(), &ec2.CreateLaunchTemplateVersionInput{ LaunchTemplateData: &ec2Types.RequestLaunchTemplateData{ - ImageId: aws.String(data.AMI), - UserData: aws.String(encodedUserData), - SecurityGroupIds: data.SecurityGroups, - InstanceType: ec2Types.InstanceType(data.InstanceType), + ImageId: aws.String(data.AMI), + UserData: aws.String(encodedUserData), + SecurityGroupIds: data.SecurityGroups, + InstanceType: ec2Types.InstanceType(data.InstanceType), + NetworkInterfaces: networkInterfaces, }, LaunchTemplateName: aws.String(data.Name), }) diff --git a/model/aws_types.go b/model/aws_types.go index 5d520603e..ac23c645e 100644 --- a/model/aws_types.go +++ b/model/aws_types.go @@ -10,10 +10,11 @@ type Certificate struct { } type LaunchTemplateData struct { - Name string - ClusterName string - AMI string - MaxPodsPerNode int64 - SecurityGroups []string - InstanceType string + Name string + ClusterName string + AMI string + MaxPodsPerNode int64 + SecurityGroups []string + InstanceType string + WithPublicSubnet bool } From 59f4b45eb57e02654f5e19baeb367dd5c8c602c0 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Thu, 4 May 2023 20:22:59 +0600 Subject: [PATCH 10/20] update --- internal/tools/aws/ec2.go | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/tools/aws/ec2.go b/internal/tools/aws/ec2.go index a5d958885..e0fda345f 100644 --- a/internal/tools/aws/ec2.go +++ b/internal/tools/aws/ec2.go @@ -186,12 +186,15 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { encodedUserData := base64.StdEncoding.EncodeToString([]byte(userData)) var networkInterfaces []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest - if data.WithPublicSubnet { - networkInterfaces = []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ - { - DeviceIndex: aws.Int32(0), - AssociatePublicIpAddress: aws.Bool(true), - }, + if data.WithPublicSubnet || len(data.SecurityGroups) > 0 { + networkInterface := ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + DeviceIndex: aws.Int32(0), + } + if len(data.SecurityGroups) > 0 { + networkInterface.Groups = data.SecurityGroups + } + if data.WithPublicSubnet { + networkInterface.AssociatePublicIpAddress = aws.Bool(data.WithPublicSubnet) } } @@ -199,7 +202,6 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { LaunchTemplateData: &ec2Types.RequestLaunchTemplateData{ ImageId: aws.String(data.AMI), UserData: aws.String(encodedUserData), - SecurityGroupIds: data.SecurityGroups, InstanceType: ec2Types.InstanceType(data.InstanceType), NetworkInterfaces: networkInterfaces, }, @@ -234,12 +236,15 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { encodedUserData := base64.StdEncoding.EncodeToString([]byte(userData)) var networkInterfaces []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest - if data.WithPublicSubnet { - networkInterfaces = []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ - { - DeviceIndex: aws.Int32(0), - AssociatePublicIpAddress: aws.Bool(true), - }, + if data.WithPublicSubnet || len(data.SecurityGroups) > 0 { + networkInterface := ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + DeviceIndex: aws.Int32(0), + } + if len(data.SecurityGroups) > 0 { + networkInterface.Groups = data.SecurityGroups + } + if data.WithPublicSubnet { + networkInterface.AssociatePublicIpAddress = aws.Bool(data.WithPublicSubnet) } } @@ -247,7 +252,6 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { LaunchTemplateData: &ec2Types.RequestLaunchTemplateData{ ImageId: aws.String(data.AMI), UserData: aws.String(encodedUserData), - SecurityGroupIds: data.SecurityGroups, InstanceType: ec2Types.InstanceType(data.InstanceType), NetworkInterfaces: networkInterfaces, }, From 344523ff341c3348ee5744b4f9e0ae17a3f4d453 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Thu, 4 May 2023 21:09:06 +0600 Subject: [PATCH 11/20] updated --- internal/tools/aws/ec2.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/tools/aws/ec2.go b/internal/tools/aws/ec2.go index e0fda345f..c56b12689 100644 --- a/internal/tools/aws/ec2.go +++ b/internal/tools/aws/ec2.go @@ -196,6 +196,7 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { if data.WithPublicSubnet { networkInterface.AssociatePublicIpAddress = aws.Bool(data.WithPublicSubnet) } + networkInterfaces = append(networkInterfaces, networkInterface) } launchTemplate, err := a.Service().ec2.CreateLaunchTemplate(context.TODO(), &ec2.CreateLaunchTemplateInput{ @@ -246,6 +247,7 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { if data.WithPublicSubnet { networkInterface.AssociatePublicIpAddress = aws.Bool(data.WithPublicSubnet) } + networkInterfaces = append(networkInterfaces, networkInterface) } launchTemplate, err := a.Service().ec2.CreateLaunchTemplateVersion(context.TODO(), &ec2.CreateLaunchTemplateVersionInput{ From 58e8ad2165a9b667f1e1ce5b3d9de1ce43f07d4d Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Fri, 5 May 2023 02:03:30 +0600 Subject: [PATCH 12/20] updated --- internal/tools/aws/ec2.go | 32 ++++++++++++++++---------------- model/aws_types.go | 4 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/internal/tools/aws/ec2.go b/internal/tools/aws/ec2.go index c56b12689..5bc4cbe03 100644 --- a/internal/tools/aws/ec2.go +++ b/internal/tools/aws/ec2.go @@ -185,7 +185,12 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { userData := getLaunchTemplateUserData(eksCluster, data) encodedUserData := base64.StdEncoding.EncodeToString([]byte(userData)) - var networkInterfaces []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest + templateData := &ec2Types.RequestLaunchTemplateData{ + ImageId: aws.String(data.AMI), + UserData: aws.String(encodedUserData), + InstanceType: ec2Types.InstanceType(data.InstanceType), + } + if data.WithPublicSubnet || len(data.SecurityGroups) > 0 { networkInterface := ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ DeviceIndex: aws.Int32(0), @@ -196,16 +201,11 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { if data.WithPublicSubnet { networkInterface.AssociatePublicIpAddress = aws.Bool(data.WithPublicSubnet) } - networkInterfaces = append(networkInterfaces, networkInterface) + templateData.NetworkInterfaces = []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{networkInterface} } launchTemplate, err := a.Service().ec2.CreateLaunchTemplate(context.TODO(), &ec2.CreateLaunchTemplateInput{ - LaunchTemplateData: &ec2Types.RequestLaunchTemplateData{ - ImageId: aws.String(data.AMI), - UserData: aws.String(encodedUserData), - InstanceType: ec2Types.InstanceType(data.InstanceType), - NetworkInterfaces: networkInterfaces, - }, + LaunchTemplateData: templateData, LaunchTemplateName: aws.String(data.Name), }) if err != nil { @@ -236,7 +236,12 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { userData := getLaunchTemplateUserData(eksCluster, data) encodedUserData := base64.StdEncoding.EncodeToString([]byte(userData)) - var networkInterfaces []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest + templateData := &ec2Types.RequestLaunchTemplateData{ + ImageId: aws.String(data.AMI), + UserData: aws.String(encodedUserData), + InstanceType: ec2Types.InstanceType(data.InstanceType), + } + if data.WithPublicSubnet || len(data.SecurityGroups) > 0 { networkInterface := ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ DeviceIndex: aws.Int32(0), @@ -247,16 +252,11 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { if data.WithPublicSubnet { networkInterface.AssociatePublicIpAddress = aws.Bool(data.WithPublicSubnet) } - networkInterfaces = append(networkInterfaces, networkInterface) + templateData.NetworkInterfaces = []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{networkInterface} } launchTemplate, err := a.Service().ec2.CreateLaunchTemplateVersion(context.TODO(), &ec2.CreateLaunchTemplateVersionInput{ - LaunchTemplateData: &ec2Types.RequestLaunchTemplateData{ - ImageId: aws.String(data.AMI), - UserData: aws.String(encodedUserData), - InstanceType: ec2Types.InstanceType(data.InstanceType), - NetworkInterfaces: networkInterfaces, - }, + LaunchTemplateData: templateData, LaunchTemplateName: aws.String(data.Name), }) if err != nil { diff --git a/model/aws_types.go b/model/aws_types.go index ac23c645e..e0970116a 100644 --- a/model/aws_types.go +++ b/model/aws_types.go @@ -12,9 +12,9 @@ type Certificate struct { type LaunchTemplateData struct { Name string ClusterName string + InstanceType string AMI string MaxPodsPerNode int64 - SecurityGroups []string - InstanceType string WithPublicSubnet bool + SecurityGroups []string } From 13d0cf5462d8ea42d422d9c1d6873ee9884feb6f Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Fri, 5 May 2023 14:24:40 +0600 Subject: [PATCH 13/20] updated --- internal/provisioner/eks_provisioner.go | 41 +++---------------------- 1 file changed, 4 insertions(+), 37 deletions(-) diff --git a/internal/provisioner/eks_provisioner.go b/internal/provisioner/eks_provisioner.go index c6b4feaeb..526dd67f5 100644 --- a/internal/provisioner/eks_provisioner.go +++ b/internal/provisioner/eks_provisioner.go @@ -475,40 +475,6 @@ func (provisioner *EKSProvisioner) RotateClusterNodes(cluster *model.Cluster) er return nil } -func (provisioner *EKSProvisioner) isMigrationRequired(oldNodeGroup *eksTypes.Nodegroup, eksMetadata *model.EKSMetadata, ngPrefix string, logger log.FieldLogger) bool { - changeRequest := eksMetadata.ChangeRequest - - if oldNodeGroup == nil { - logger.Debug("Old EKS NodeGroup is nil") - return false - } - - ngChangeRequest, found := changeRequest.NodeGroups[ngPrefix] - if !found { - logger.Debugf("NodeGroup %s not found in ChangeRequest", ngPrefix) - return false - } - - if changeRequest.MaxPodsPerNode > 0 || changeRequest.AMI != "" { - return true - } - - if oldNodeGroup.InstanceTypes != nil && len(oldNodeGroup.InstanceTypes) > 0 && - oldNodeGroup.InstanceTypes[0] != ngChangeRequest.InstanceType { - return true - } - - scalingInfo := oldNodeGroup.ScalingConfig - if *scalingInfo.MinSize != int32(ngChangeRequest.MinCount) { - return true - } - if *scalingInfo.MaxSize != int32(ngChangeRequest.MaxCount) { - return true - } - - return false -} - // ResizeCluster resizes cluster - not implemented. func (provisioner *EKSProvisioner) ResizeCluster(cluster *model.Cluster) error { logger := provisioner.logger.WithField("cluster", cluster.ID) @@ -546,9 +512,10 @@ func (provisioner *EKSProvisioner) ResizeCluster(cluster *model.Cluster) error { return } - if !provisioner.isMigrationRequired(oldEKSNodeGroup, eksMetadata, ngPrefix, logger) { - logger.Debugf("EKS NodeGroup migration not required for %s", ngPrefix) - return + if oldMetadata.InstanceType == ngMetadata.InstanceType && + oldMetadata.MinCount == ngMetadata.MinCount && + oldMetadata.MaxCount == ngMetadata.MaxCount { + logger.Debugf("No change in EKS NodeGroup for %s", ngPrefix) } err = provisioner.prepareLaunchTemplate(cluster, ngPrefix, ngMetadata, logger) From 43d5013f862cc236bda41fb5c4a98e7202af3fc1 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Sat, 6 May 2023 01:23:44 +0600 Subject: [PATCH 14/20] updated --- internal/tools/aws/ec2.go | 47 +++++++++++++++---------------- internal/tools/aws/eks.go | 58 +++++++++++++++++++++++---------------- 2 files changed, 57 insertions(+), 48 deletions(-) diff --git a/internal/tools/aws/ec2.go b/internal/tools/aws/ec2.go index 5bc4cbe03..77d5596b4 100644 --- a/internal/tools/aws/ec2.go +++ b/internal/tools/aws/ec2.go @@ -187,21 +187,19 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { templateData := &ec2Types.RequestLaunchTemplateData{ ImageId: aws.String(data.AMI), - UserData: aws.String(encodedUserData), InstanceType: ec2Types.InstanceType(data.InstanceType), + UserData: aws.String(encodedUserData), + NetworkInterfaces: []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + { + AssociatePublicIpAddress: aws.Bool(data.WithPublicSubnet), + DeleteOnTermination: aws.Bool(true), + DeviceIndex: aws.Int32(0), + }, + }, } - if data.WithPublicSubnet || len(data.SecurityGroups) > 0 { - networkInterface := ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ - DeviceIndex: aws.Int32(0), - } - if len(data.SecurityGroups) > 0 { - networkInterface.Groups = data.SecurityGroups - } - if data.WithPublicSubnet { - networkInterface.AssociatePublicIpAddress = aws.Bool(data.WithPublicSubnet) - } - templateData.NetworkInterfaces = []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{networkInterface} + for _, sg := range data.SecurityGroups { + templateData.NetworkInterfaces[0].Groups = append(templateData.NetworkInterfaces[0].Groups, sg) } launchTemplate, err := a.Service().ec2.CreateLaunchTemplate(context.TODO(), &ec2.CreateLaunchTemplateInput{ @@ -238,21 +236,19 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { templateData := &ec2Types.RequestLaunchTemplateData{ ImageId: aws.String(data.AMI), - UserData: aws.String(encodedUserData), InstanceType: ec2Types.InstanceType(data.InstanceType), + UserData: aws.String(encodedUserData), + NetworkInterfaces: []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + { + AssociatePublicIpAddress: aws.Bool(data.WithPublicSubnet), + DeleteOnTermination: aws.Bool(true), + DeviceIndex: aws.Int32(0), + }, + }, } - if data.WithPublicSubnet || len(data.SecurityGroups) > 0 { - networkInterface := ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ - DeviceIndex: aws.Int32(0), - } - if len(data.SecurityGroups) > 0 { - networkInterface.Groups = data.SecurityGroups - } - if data.WithPublicSubnet { - networkInterface.AssociatePublicIpAddress = aws.Bool(data.WithPublicSubnet) - } - templateData.NetworkInterfaces = []ec2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{networkInterface} + for _, sg := range data.SecurityGroups { + templateData.NetworkInterfaces[0].Groups = append(templateData.NetworkInterfaces[0].Groups, sg) } launchTemplate, err := a.Service().ec2.CreateLaunchTemplateVersion(context.TODO(), &ec2.CreateLaunchTemplateVersionInput{ @@ -329,6 +325,7 @@ func getLaunchTemplateUserData(eksCluster *eksTypes.Cluster, data *model.LaunchT dataTemplate := ` #!/bin/bash set -o xtrace -/etc/eks/bootstrap.sh '%s' --apiserver-endpoint '%s' --b64-cluster-ca '%s' --use-max-pods false --kubelet-extra-args '--max-pods=%d'` +/etc/eks/bootstrap.sh '%s' --apiserver-endpoint '%s' --b64-cluster-ca '%s' --use-max-pods false --kubelet-extra-args '--max-pods=%d --kube-reserved cpu=250m,memory=1Gi,ephemeral-storage=1Gi --system-reserved cpu=250m,memory=0.2Gi,ephemeral-storage=1Gi --eviction-hard memory.available<0.2Gi,nodefs.available<10%%' +` return fmt.Sprintf(dataTemplate, *eksCluster.Name, *eksCluster.Endpoint, *eksCluster.CertificateAuthority.Data, data.MaxPodsPerNode) } diff --git a/internal/tools/aws/eks.go b/internal/tools/aws/eks.go index 3642757d3..00d3b7dc3 100644 --- a/internal/tools/aws/eks.go +++ b/internal/tools/aws/eks.go @@ -113,32 +113,44 @@ func (c *Client) EnsureEKSCluster(cluster *model.Cluster, resources ClusterResou // InstallEKSAddons installs EKS EBS addon to the existing cluster. func (a *Client) InstallEKSAddons(cluster *model.Cluster) error { - input := eks.CreateAddonInput{ - AddonName: aws.String("aws-ebs-csi-driver"), - ClusterName: aws.String(cluster.ProvisionerMetadataEKS.Name), - } - _, err := a.Service().eks.CreateAddon(context.TODO(), &input) - if err != nil { - // In case addon already configured we do not want to fail. - if IsErrorResourceInUseException(err) { - return nil - } - return errors.Wrap(err, "failed to create ebs-csi addon") - } - input = eks.CreateAddonInput{ - AddonName: aws.String("vpc-cni"), - ClusterName: aws.String(cluster.ProvisionerMetadataEKS.Name), - ConfigurationValues: aws.String("{\"env\":{\"ENABLE_PREFIX_DELEGATION\":\"true\"}}"), - ResolveConflicts: eksTypes.ResolveConflictsOverwrite, + var addons = []eks.CreateAddonInput{ + { + AddonName: aws.String("coredns"), + AddonVersion: aws.String("v1.8.7-eksbuild.1"), + ClusterName: aws.String(cluster.ProvisionerMetadataEKS.Name), + ResolveConflicts: eksTypes.ResolveConflictsOverwrite, + }, + { + AddonName: aws.String("kube-proxy"), + AddonVersion: aws.String("v1.22.6-eksbuild.1"), + ClusterName: aws.String(cluster.ProvisionerMetadataEKS.Name), + ResolveConflicts: eksTypes.ResolveConflictsOverwrite, + }, + { + AddonName: aws.String("vpc-cni"), + AddonVersion: aws.String("v1.11.0-eksbuild.1"), + ClusterName: aws.String(cluster.ProvisionerMetadataEKS.Name), + ConfigurationValues: aws.String("{\"env\":{\"ENABLE_PREFIX_DELEGATION\":\"true\"}}"), + ResolveConflicts: eksTypes.ResolveConflictsOverwrite, + }, + { + AddonName: aws.String("aws-ebs-csi-driver"), + AddonVersion: aws.String("v1.11.2-eksbuild.1"), + ClusterName: aws.String(cluster.ProvisionerMetadataEKS.Name), + ResolveConflicts: eksTypes.ResolveConflictsOverwrite, + }, } - _, err = a.Service().eks.CreateAddon(context.TODO(), &input) - if err != nil { - // In case addon already configured we do not want to fail. - if IsErrorResourceInUseException(err) { - return nil + + for i, addon := range addons { + _, err := a.Service().eks.CreateAddon(context.TODO(), &addons[i]) + if err != nil { + // In case addon already configured we do not want to fail. + if IsErrorResourceInUseException(err) { + return nil + } + return errors.Wrapf(err, "failed to create %s addon", *addon.AddonName) } - return errors.Wrap(err, "failed to create vpc-cni addon") } return nil From 4cd48672fd382e25822ff459746ed20e4e772532 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Mon, 8 May 2023 10:08:51 +0600 Subject: [PATCH 15/20] updated --- internal/tools/aws/eks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/tools/aws/eks.go b/internal/tools/aws/eks.go index 00d3b7dc3..e2eed50b6 100644 --- a/internal/tools/aws/eks.go +++ b/internal/tools/aws/eks.go @@ -537,7 +537,7 @@ func (c *Client) WaitForEKSNodeGroupToBeDeleted(clusterName, workerName string, for { select { case <-timeoutTimer.C: - return errors.New("timed out waiting for EKS NodeGroup to become ready") + return errors.New("timed out waiting for EKS NodeGroup to be deleted") case <-tick.C: nodeGroup, err := c.getEKSNodeGroup(clusterName, workerName) if err != nil { From a2b52e0e525c57e765af385ff2968cab650ec972 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Mon, 8 May 2023 23:24:20 +0600 Subject: [PATCH 16/20] support nodegroup size customization --- clusterdictionary/size.go | 65 ++++++++++++++++++++++++++++++--------- go.mod | 13 ++++---- go.sum | 29 +++++++++-------- internal/tools/aws/ec2.go | 10 ++---- model/cluster_request.go | 4 +-- 5 files changed, 79 insertions(+), 42 deletions(-) diff --git a/clusterdictionary/size.go b/clusterdictionary/size.go index 153f738f7..a6cd54029 100644 --- a/clusterdictionary/size.go +++ b/clusterdictionary/size.go @@ -5,6 +5,11 @@ package clusterdictionary import ( + "strconv" + "strings" + + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/gookit/goutil/arrutil" "github.com/mattermost/mattermost-cloud/model" "github.com/pkg/errors" ) @@ -130,6 +135,40 @@ func ApplyToPatchClusterSizeRequest(size string, request *model.PatchClusterSize return nil } +func processCustomSize(size string) (string, int64, int64, error) { + if len(size) == 0 { + return "", 0, 0, nil + } + + parts := strings.Split(size, ";") + ngType := ec2Types.InstanceType(parts[0]) + + if !arrutil.In[ec2Types.InstanceType](ngType, ngType.Values()) { + return "", 0, 0, errors.Errorf("%s is not a valid node group type", ngType) + } + minCount := 2 + maxCount := 2 + + for _, part := range parts[1:] { + switch { + case strings.HasPrefix(part, "min="): + minCount, _ = strconv.Atoi(strings.TrimPrefix(part, "min=")) + case strings.HasPrefix(part, "max="): + maxCount, _ = strconv.Atoi(strings.TrimPrefix(part, "max=")) + } + } + + if minCount < 1 { + minCount = 1 + } + + if minCount > maxCount { + maxCount = minCount + } + + return string(ngType), int64(minCount), int64(maxCount), nil +} + // AddToCreateClusterRequest takes a map of size keywords and adds the corresponding // values to a CreateClusterRequest. func AddToCreateClusterRequest(sizes map[string]string, request *model.CreateClusterRequest) error { @@ -142,17 +181,16 @@ func AddToCreateClusterRequest(sizes map[string]string, request *model.CreateClu } for ng, ngSize := range sizes { - if !IsValidClusterSize(ngSize) { - return errors.Errorf("%s is not a valid size", ngSize) + ngType, minCount, maxCount, err := processCustomSize(ngSize) + if err != nil { + return errors.Wrapf(err, "invalid nodegroup size for %s", ng) } - values := ValidSizes[ngSize] - request.AdditionalNodeGroups[ng] = model.NodeGroupMetadata{ // These values are used in EKS configuration, but not in kops. - InstanceType: values.NodeInstanceType, - MinCount: values.NodeMinCount, - MaxCount: values.NodeMaxCount, + InstanceType: ngType, + MinCount: minCount, + MaxCount: maxCount, } } @@ -171,16 +209,15 @@ func AddToCreateNodegroupsRequest(sizes map[string]string, request *model.Create } for ng, ngSize := range sizes { - if !IsValidClusterSize(ngSize) { - return errors.Errorf("%s is not a valid size", ngSize) + ngType, minCount, maxCount, err := processCustomSize(ngSize) + if err != nil { + return errors.Wrapf(err, "invalid nodegroup size for %s", ng) } - values := ValidSizes[ngSize] - request.Nodegroups[ng] = model.NodeGroupMetadata{ - InstanceType: values.NodeInstanceType, - MinCount: values.NodeMinCount, - MaxCount: values.NodeMaxCount, + InstanceType: ngType, + MinCount: minCount, + MaxCount: maxCount, } } diff --git a/go.mod b/go.mod index 9a8548fc4..09226c425 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/cloudflare/cloudflare-go v0.54.0 github.com/go-sql-driver/mysql v1.6.0 github.com/golang/mock v1.6.0 + github.com/gookit/goutil v0.6.8 github.com/gorilla/mux v1.8.0 github.com/gosuri/uilive v0.0.4 github.com/jmoiron/sqlx v1.3.5 @@ -49,7 +50,7 @@ require ( github.com/spf13/viper v1.14.0 github.com/stretchr/testify v1.8.1 github.com/vrischmann/envconfig v1.3.0 - golang.org/x/tools v0.2.0 + golang.org/x/tools v0.6.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.23.0 k8s.io/apiextensions-apiserver v0.23.0 @@ -130,12 +131,12 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.1.0 // indirect - golang.org/x/mod v0.6.0 // indirect - golang.org/x/net v0.1.0 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.6.0 // indirect golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/term v0.1.0 // indirect - golang.org/x/text v0.4.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index e82ac5c77..538412f42 100644 --- a/go.sum +++ b/go.sum @@ -767,6 +767,9 @@ github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2c github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gookit/color v1.5.3 h1:twfIhZs4QLCtimkP7MOxlF3A0U/5cDPseRT9M/+2SCE= +github.com/gookit/goutil v0.6.8 h1:B2XXSCGav5TXWtKRT9i/s/owOLXXB7sY6UsfqeSLroE= +github.com/gookit/goutil v0.6.8/go.mod h1:u+Isykc6RQcZ4GQzulsaGm+Famd97U5Tzp3aQyo+jyA= github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gophercloud/gophercloud v0.10.0/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss= @@ -1492,6 +1495,7 @@ github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHM github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= @@ -1667,8 +1671,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1742,8 +1746,8 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1781,6 +1785,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1896,15 +1901,15 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1916,8 +1921,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2021,8 +2026,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/tools/aws/ec2.go b/internal/tools/aws/ec2.go index 77d5596b4..012947086 100644 --- a/internal/tools/aws/ec2.go +++ b/internal/tools/aws/ec2.go @@ -194,14 +194,11 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { AssociatePublicIpAddress: aws.Bool(data.WithPublicSubnet), DeleteOnTermination: aws.Bool(true), DeviceIndex: aws.Int32(0), + Groups: data.SecurityGroups, }, }, } - for _, sg := range data.SecurityGroups { - templateData.NetworkInterfaces[0].Groups = append(templateData.NetworkInterfaces[0].Groups, sg) - } - launchTemplate, err := a.Service().ec2.CreateLaunchTemplate(context.TODO(), &ec2.CreateLaunchTemplateInput{ LaunchTemplateData: templateData, LaunchTemplateName: aws.String(data.Name), @@ -243,14 +240,11 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { AssociatePublicIpAddress: aws.Bool(data.WithPublicSubnet), DeleteOnTermination: aws.Bool(true), DeviceIndex: aws.Int32(0), + Groups: data.SecurityGroups, }, }, } - for _, sg := range data.SecurityGroups { - templateData.NetworkInterfaces[0].Groups = append(templateData.NetworkInterfaces[0].Groups, sg) - } - launchTemplate, err := a.Service().ec2.CreateLaunchTemplateVersion(context.TODO(), &ec2.CreateLaunchTemplateVersionInput{ LaunchTemplateData: templateData, LaunchTemplateName: aws.String(data.Name), diff --git a/model/cluster_request.go b/model/cluster_request.go index f64dbd39e..6bc1b7ab0 100644 --- a/model/cluster_request.go +++ b/model/cluster_request.go @@ -183,8 +183,8 @@ func (request *CreateClusterRequest) Validate() error { if ng.MinCount < 1 { return errors.Errorf("node min count (%d) must be 1 or greater for node group %s", ng.MinCount, name) } - if ng.MaxCount != ng.MinCount { - return errors.Errorf("node min (%d) and max (%d) counts must match for node group %s", ng.MinCount, ng.MaxCount, name) + if ng.MinCount > ng.MaxCount { + return errors.Errorf("node min count (%d) cannot be greater than max count (%d) for node group %s", ng.MinCount, ng.MaxCount, name) } } } From 4b4bdd48cd4d3ab635b072a275a6f9588b6ef884 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Tue, 9 May 2023 13:55:51 +0600 Subject: [PATCH 17/20] updated --- clusterdictionary/size.go | 3 +- internal/api/cluster.go | 61 --------------------------------------- model/cluster_request.go | 2 +- 3 files changed, 3 insertions(+), 63 deletions(-) diff --git a/clusterdictionary/size.go b/clusterdictionary/size.go index a6cd54029..b5de8036f 100644 --- a/clusterdictionary/size.go +++ b/clusterdictionary/size.go @@ -144,8 +144,9 @@ func processCustomSize(size string) (string, int64, int64, error) { ngType := ec2Types.InstanceType(parts[0]) if !arrutil.In[ec2Types.InstanceType](ngType, ngType.Values()) { - return "", 0, 0, errors.Errorf("%s is not a valid node group type", ngType) + return "", 0, 0, errors.Errorf("%s is not a valid InstanceType", ngType) } + minCount := 2 maxCount := 2 diff --git a/internal/api/cluster.go b/internal/api/cluster.go index a282171dd..044307c18 100644 --- a/internal/api/cluster.go +++ b/internal/api/cluster.go @@ -558,67 +558,6 @@ func handleDeleteNodegroup(c *Context, w http.ResponseWriter, r *http.Request) { outputJSON(c, w, clusterDTO) } -// handleDeleteNodegroup responds to DELETE /api/cluster/{cluster}/nodegroup/{nodegroup}, deleting -// the requested nodegroup. -func handleDeleteNodegroup(c *Context, w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - clusterID := vars["cluster"] - nodegroup := vars["nodegroup"] - c.Logger = c.Logger.WithField("cluster", clusterID) - - newState := model.ClusterStateNodegroupsDeletionRequested - - clusterDTO, status, unlockOnce := getClusterForTransition(c, clusterID, newState) - if status != 0 { - c.Logger.Debug("Cluster is not in a valid state for nodegroup deletion") - w.WriteHeader(status) - return - } - defer unlockOnce() - - if clusterDTO.Provisioner == model.ProvisionerKops { - c.Logger.Debug("Deleting nodegroup for Kops cluster is not supported") - w.WriteHeader(http.StatusBadRequest) - return - } - - if clusterDTO.Provisioner == model.ProvisionerEKS { - err := clusterDTO.ProvisionerMetadataEKS.ValidateNodegroupDeleteRequest(nodegroup) - if err != nil { - c.Logger.WithError(err).Error("failed to validate nodegroup delete request") - w.WriteHeader(http.StatusBadRequest) - return - } - - // Set Nodegroup in change request - clusterDTO.ProvisionerMetadataEKS.ApplyNodegroupDeleteRequest(nodegroup) - } - - oldState := clusterDTO.State - - clusterDTO.State = newState - err := c.Store.UpdateCluster(clusterDTO.Cluster) - if err != nil { - c.Logger.WithError(err).Error("failed to update cluster") - w.WriteHeader(http.StatusInternalServerError) - return - } - - if oldState != newState { - err = c.EventProducer.ProduceClusterStateChangeEvent(clusterDTO.Cluster, oldState) - if err != nil { - c.Logger.WithError(err).Error("failed to create cluster state change event") - } - } - - unlockOnce() - _ = c.Supervisor.Do() - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusAccepted) - outputJSON(c, w, clusterDTO) -} - // handleDeleteCluster responds to DELETE /api/cluster/{cluster}, beginning the process of // deleting the cluster. func handleDeleteCluster(c *Context, w http.ResponseWriter, r *http.Request) { diff --git a/model/cluster_request.go b/model/cluster_request.go index 6bc1b7ab0..966d2284d 100644 --- a/model/cluster_request.go +++ b/model/cluster_request.go @@ -184,7 +184,7 @@ func (request *CreateClusterRequest) Validate() error { return errors.Errorf("node min count (%d) must be 1 or greater for node group %s", ng.MinCount, name) } if ng.MinCount > ng.MaxCount { - return errors.Errorf("node min count (%d) cannot be greater than max count (%d) for node group %s", ng.MinCount, ng.MaxCount, name) + return errors.Errorf("node min count (%d) cannot be greater than max count (%d) for nodegroup %s", ng.MinCount, ng.MaxCount, name) } } } From ac7cef922ff9aabea68ea6c40a6df1d49d3cd49b Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Tue, 9 May 2023 14:00:27 +0600 Subject: [PATCH 18/20] add metrics for nodegroup tasks --- internal/metrics/metrics.go | 32 +++++++++++++++++++++++++++----- internal/supervisor/cluster.go | 6 ++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 871f0635d..9a39fbbd9 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -36,11 +36,13 @@ type CloudMetrics struct { ClusterInstallationDeletionDurationHist *prometheus.HistogramVec // Cluster - ClusterCreationDurationHist *prometheus.HistogramVec - ClusterUpgradeDurationHist *prometheus.HistogramVec - ClusterProvisioningDurationHist *prometheus.HistogramVec - ClusterResizeDurationHist *prometheus.HistogramVec - ClusterDeletionDurationHist *prometheus.HistogramVec + ClusterCreationDurationHist *prometheus.HistogramVec + ClusterUpgradeDurationHist *prometheus.HistogramVec + ClusterProvisioningDurationHist *prometheus.HistogramVec + ClusterResizeDurationHist *prometheus.HistogramVec + ClusterDeletionDurationHist *prometheus.HistogramVec + ClusterNodegroupsCreationDurationHist *prometheus.HistogramVec + ClusterNodegroupsDeletionDurationHist *prometheus.HistogramVec } // New creates a new Prometheus-based Metrics object to be used @@ -191,6 +193,26 @@ func New() *CloudMetrics { }, []string{}, ), + ClusterNodegroupsCreationDurationHist: promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: provisionerNamespace, + Subsystem: provisionerSubsystemApp, + Name: "cluster_nodegroups_creation_duration_seconds", + Help: "The duration of cluster nodegroups creation tasks", + Buckets: standardDurationBuckets(), + }, + []string{}, + ), + ClusterNodegroupsDeletionDurationHist: promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: provisionerNamespace, + Subsystem: provisionerSubsystemApp, + Name: "cluster_nodegroups_deletion_duration_seconds", + Help: "The duration of cluster nodegroups deletion tasks", + Buckets: standardDurationBuckets(), + }, + []string{}, + ), } } diff --git a/internal/supervisor/cluster.go b/internal/supervisor/cluster.go index 6ac370f3b..ff8baf073 100644 --- a/internal/supervisor/cluster.go +++ b/internal/supervisor/cluster.go @@ -390,6 +390,12 @@ func (s *ClusterSupervisor) processClusterMetrics(cluster *model.Cluster, logger case model.ClusterStateDeletionRequested: s.metrics.ClusterDeletionDurationHist.WithLabelValues().Observe(elapsedSeconds) logger.Debugf("Cluster was deleted in %d seconds", int(elapsedSeconds)) + case model.ClusterStateNodegroupsCreationRequested: + s.metrics.ClusterNodegroupsCreationDurationHist.WithLabelValues().Observe(elapsedSeconds) + logger.Debugf("Cluster nodegroups were created in %d seconds", int(elapsedSeconds)) + case model.ClusterStateNodegroupsDeletionRequested: + s.metrics.ClusterNodegroupsDeletionDurationHist.WithLabelValues().Observe(elapsedSeconds) + logger.Debugf("Cluster nodegroups were deleted in %d seconds", int(elapsedSeconds)) default: return errors.Errorf("failed to handle event %s with new state %s", event.Event.ID, event.StateChange.NewState) } From fc7a6b04e5de9bc786dc55d69b6ac4bfcc754671 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Tue, 9 May 2023 17:04:06 +0600 Subject: [PATCH 19/20] updated error & log messages --- clusterdictionary/size.go | 1 - cmd/cloud/cluster.go | 4 +- cmd/cloud/cluster_flag.go | 2 +- cmd/cloud/nodegroup_flag.go | 6 +- internal/api/cluster.go | 14 +-- internal/provisioner/eks_provisioner.go | 134 ++++++++++++------------ internal/supervisor/cluster.go | 24 ++--- internal/tools/aws/ec2.go | 18 ++-- internal/tools/aws/eks.go | 88 ++++++++-------- model/cluster_request.go | 8 +- model/eks_metadata.go | 12 +-- 11 files changed, 154 insertions(+), 157 deletions(-) diff --git a/clusterdictionary/size.go b/clusterdictionary/size.go index b5de8036f..865c9cb23 100644 --- a/clusterdictionary/size.go +++ b/clusterdictionary/size.go @@ -188,7 +188,6 @@ func AddToCreateClusterRequest(sizes map[string]string, request *model.CreateClu } request.AdditionalNodeGroups[ng] = model.NodeGroupMetadata{ - // These values are used in EKS configuration, but not in kops. InstanceType: ngType, MinCount: minCount, MaxCount: maxCount, diff --git a/cmd/cloud/cluster.go b/cmd/cloud/cluster.go index 07084beb0..b9b5bdc60 100644 --- a/cmd/cloud/cluster.go +++ b/cmd/cloud/cluster.go @@ -136,7 +136,7 @@ func executeClusterCreateCmd(flags clusterCreateFlags) error { continue } if _, f := flags.additionalNodegroups[ng]; !f { - return fmt.Errorf("nodegroup %s not provided as additional nodegroups", ng) + return fmt.Errorf("nodegroup %s not provided as an additional nodegroup", ng) } } @@ -145,7 +145,7 @@ func executeClusterCreateCmd(flags clusterCreateFlags) error { continue } if _, f := flags.additionalNodegroups[ng]; !f { - return fmt.Errorf("nodegroup %s not provided as additional nodegroups", ng) + return fmt.Errorf("nodegroup %s not provided as an additional nodegroup", ng) } } diff --git a/cmd/cloud/cluster_flag.go b/cmd/cloud/cluster_flag.go index a5cd92ecb..7aef53652 100644 --- a/cmd/cloud/cluster_flag.go +++ b/cmd/cloud/cluster_flag.go @@ -49,7 +49,7 @@ func (flags *createRequestOptions) addFlags(command *cobra.Command) { command.Flags().StringVar(&flags.clusterRoleARN, "cluster-role-arn", "", "AWS role ARN for cluster.") command.Flags().StringVar(&flags.nodeRoleARN, "node-role-arn", "", "AWS role ARN for node.") command.Flags().BoolVar(&flags.useEKS, "eks", false, "Create EKS cluster.") - command.Flags().StringToStringVar(&flags.additionalNodegroups, "additional-nodegroups", nil, "Additional nodegroups to create. The key is the name of the nodegroup and the value is the size constant.") + command.Flags().StringToStringVar(&flags.additionalNodegroups, "additional-nodegroups", nil, "Additional nodegroups to create. Example: --additional-nodegroups ng1=t3.medium;min=1;max=3") command.Flags().StringSliceVar(&flags.nodegroupsWithPublicSubnet, "nodegroups-with-public-subnet", nil, "Nodegroups to create with public subnet. The value is the name of the nodegroup.") command.Flags().StringSliceVar(&flags.nodegroupsWithSecurityGroup, "nodegroups-with-sg", nil, "Nodegroups to create with dedicated security group. The value is the name of the nodegroup.") } diff --git a/cmd/cloud/nodegroup_flag.go b/cmd/cloud/nodegroup_flag.go index 059e6fa77..6d42ad309 100644 --- a/cmd/cloud/nodegroup_flag.go +++ b/cmd/cloud/nodegroup_flag.go @@ -16,9 +16,9 @@ type clusterNodegroupsCreateFlags struct { func (flags *clusterNodegroupsCreateFlags) addFlags(command *cobra.Command) { command.Flags().StringVar(&flags.clusterID, "cluster", "", "The id of the cluster to be modified.") - command.Flags().StringToStringVar(&flags.nodegroups, "nodegroups", nil, "Additional nodegroups to create. The key is the name of the nodegroup and the value is the size constant.") - command.Flags().StringSliceVar(&flags.nodegroupsWithPublicSubnet, "nodegroups-with-public-subnet", nil, "Nodegroups to create with public subnet. The value is the name of the nodegroup.") - command.Flags().StringSliceVar(&flags.nodegroupsWithSecurityGroup, "nodegroups-with-sg", nil, "Nodegroups to create with dedicated security group. The value is the name of the nodegroup.") + command.Flags().StringToStringVar(&flags.nodegroups, "nodegroups", nil, "Additional nodegroups to create. Example: --nodegroups ng1=t3.medium;min=1;max=3") + command.Flags().StringSliceVar(&flags.nodegroupsWithPublicSubnet, "nodegroups-with-public-subnet", nil, "Nodegroups to create with public subnet. The value is the name of the nodegroups.") + command.Flags().StringSliceVar(&flags.nodegroupsWithSecurityGroup, "nodegroups-with-sg", nil, "Nodegroups to create with dedicated security group. The value is the name of the nodegroupa.") _ = command.MarkFlagRequired("cluster") _ = command.MarkFlagRequired("nodegroups") diff --git a/internal/api/cluster.go b/internal/api/cluster.go index 044307c18..35161e2bd 100644 --- a/internal/api/cluster.go +++ b/internal/api/cluster.go @@ -193,7 +193,7 @@ func handleRetryCreateCluster(c *Context, w http.ResponseWriter, r *http.Request // Notify even if we didn't make changes, to expedite even the no-op operations above. unlockOnce() - c.Supervisor.Do() + _ = c.Supervisor.Do() w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) @@ -249,7 +249,7 @@ func handleProvisionCluster(c *Context, w http.ResponseWriter, r *http.Request) // Notify even if we didn't make changes, to expedite even the no-op operations above. unlockOnce() - c.Supervisor.Do() + _ = c.Supervisor.Do() w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) @@ -350,7 +350,7 @@ func handleUpgradeKubernetes(c *Context, w http.ResponseWriter, r *http.Request) } unlockOnce() - c.Supervisor.Do() + _ = c.Supervisor.Do() w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) @@ -423,7 +423,7 @@ func handleResizeCluster(c *Context, w http.ResponseWriter, r *http.Request) { } unlockOnce() - c.Supervisor.Do() + _ = c.Supervisor.Do() w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) @@ -463,7 +463,7 @@ func handleCreateNodegroups(c *Context, w http.ResponseWriter, r *http.Request) if clusterDTO.Provisioner == model.ProvisionerEKS { err = clusterDTO.ProvisionerMetadataEKS.ValidateNodegroupsCreateRequest(createNodegroupsRequest.Nodegroups) if err != nil { - c.Logger.WithError(err).Error("failed to validate nodegroups create request") + c.Logger.WithError(err).Error("Failed to validate nodegroups creation request") w.WriteHeader(http.StatusBadRequest) return } @@ -524,7 +524,7 @@ func handleDeleteNodegroup(c *Context, w http.ResponseWriter, r *http.Request) { if clusterDTO.Provisioner == model.ProvisionerEKS { err := clusterDTO.ProvisionerMetadataEKS.ValidateNodegroupDeleteRequest(nodegroup) if err != nil { - c.Logger.WithError(err).Error("failed to validate nodegroup delete request") + c.Logger.WithError(err).Error("Failed to validate nodegroup deletion request") w.WriteHeader(http.StatusBadRequest) return } @@ -636,7 +636,7 @@ func handleDeleteCluster(c *Context, w http.ResponseWriter, r *http.Request) { } unlockOnce() - c.Supervisor.Do() + _ = c.Supervisor.Do() w.WriteHeader(http.StatusAccepted) } diff --git a/internal/provisioner/eks_provisioner.go b/internal/provisioner/eks_provisioner.go index c9a483559..8c841ab2c 100644 --- a/internal/provisioner/eks_provisioner.go +++ b/internal/provisioner/eks_provisioner.go @@ -77,22 +77,22 @@ func (provisioner *EKSProvisioner) CreateCluster(cluster *model.Cluster) error { eksMetadata := cluster.ProvisionerMetadataEKS if eksMetadata == nil { - return errors.New("EKS Metadata not set when creating EKS cluster") + return errors.New("metadata not found to create EKS cluster") } err := eksMetadata.ValidateChangeRequest() if err != nil { - return errors.Wrap(err, "EKS Metadata ChangeRequest failed validation") + return errors.Wrap(err, "failed to validate ChangeRequest for EKS cluster") } if eksMetadata.ChangeRequest.AMI != "" && eksMetadata.ChangeRequest.AMI != "latest" { var isAMIValid bool isAMIValid, err = provisioner.awsClient.IsValidAMI(eksMetadata.ChangeRequest.AMI, logger) if err != nil { - return errors.Wrapf(err, "error checking the AWS AMI image %s", eksMetadata.ChangeRequest.AMI) + return errors.Wrapf(err, "failed to validate AMI %s", eksMetadata.ChangeRequest.AMI) } if !isAMIValid { - return errors.Errorf("invalid AWS AMI image %s", eksMetadata.ChangeRequest.AMI) + return errors.Errorf("provided AMI %s is not valid", eksMetadata.ChangeRequest.AMI) } } @@ -102,12 +102,12 @@ func (provisioner *EKSProvisioner) CreateCluster(cluster *model.Cluster) error { if eksMetadata.ChangeRequest.VPC != "" && provisioner.params.UseExistingAWSResources { clusterResources, err = provisioner.awsClient.ClaimVPC(eksMetadata.ChangeRequest.VPC, cluster, provisioner.params.Owner, logger) if err != nil { - return errors.Wrap(err, "couldn't claim VPC") + return errors.Wrap(err, "failed to claim VPC") } } else if provisioner.params.UseExistingAWSResources { clusterResources, err = provisioner.awsClient.GetAndClaimVpcResources(cluster, provisioner.params.Owner, logger) if err != nil { - return err + return errors.Wrap(err, "failed to get and claim VPC resources") } } @@ -118,19 +118,19 @@ func (provisioner *EKSProvisioner) CreateCluster(cluster *model.Cluster) error { if err != nil { releaseErr := provisioner.awsClient.ReleaseVpc(cluster, logger) if releaseErr != nil { - logger.WithError(releaseErr).Error("Unable to release VPC") + logger.WithError(releaseErr).Error("Failed to release VPC") } - return errors.Wrap(err, "unable to create EKS cluster") + return errors.Wrap(err, "failed to ensure EKS cluster") } err = provisioner.clusterUpdateStore.UpdateCluster(cluster) if err != nil { releaseErr := provisioner.awsClient.ReleaseVpc(cluster, logger) if releaseErr != nil { - logger.WithError(releaseErr).Error("Failed to release VPC after failed update") + logger.WithError(releaseErr).Error("Failed to release VPC") } - return errors.Wrap(err, "failed to update EKS metadata with VPC ID") + return errors.Wrap(err, "failed to update cluster metadata after creating EKS cluster") } return nil @@ -141,7 +141,7 @@ func (provisioner *EKSProvisioner) CheckClusterCreated(cluster *model.Cluster) ( logger := provisioner.logger.WithField("cluster", cluster.ID) if cluster.ProvisionerMetadataEKS == nil { - return false, errors.New("expected EKS metadata not to be nil") + return false, errors.New("metadata not found to check EKS cluster") } eksMetadata := cluster.ProvisionerMetadataEKS @@ -164,7 +164,7 @@ func (provisioner *EKSProvisioner) CheckClusterCreated(cluster *model.Cluster) ( err = provisioner.clusterUpdateStore.UpdateCluster(cluster) if err != nil { - return false, errors.Wrap(err, "failed to store cluster") + return false, errors.Wrap(err, "failed to save updated cluster metadata") } return true, nil @@ -184,7 +184,7 @@ func (provisioner *EKSProvisioner) prepareLaunchTemplate(cluster *model.Cluster, } securityGroups, err = provisioner.awsClient.ClaimSecurityGroups(cluster, ngPrefix, vpc, logger) if err != nil { - return errors.Wrap(err, "failed to get security groups") + return errors.Wrap(err, "failed to claim security groups") } } @@ -208,7 +208,7 @@ func (provisioner *EKSProvisioner) prepareLaunchTemplate(cluster *model.Cluster, launchTemplateName := fmt.Sprintf("%s-%s", eksMetadata.Name, ngPrefix) isAvailable, err := provisioner.awsClient.IsLaunchTemplateAvailable(launchTemplateName) if err != nil { - return errors.Wrap(err, "failed to check launch template availability") + return errors.Wrap(err, "failed to check if launch template is available") } if isAvailable { @@ -232,7 +232,7 @@ func (provisioner *EKSProvisioner) CreateNodegroups(cluster *model.Cluster) erro eksMetadata := cluster.ProvisionerMetadataEKS if eksMetadata == nil { - return errors.New("EKS Metadata not set when creating EKS NodeGroup") + return errors.New("metadata not found to create EKS nodegroups") } eksMetadata = provisioner.setMissingChangeRequest(eksMetadata) @@ -246,7 +246,7 @@ func (provisioner *EKSProvisioner) CreateNodegroups(cluster *model.Cluster) erro wg.Add(1) go func(ngPrefix string, ngMetadata model.NodeGroupMetadata) { defer wg.Done() - logger.Debugf("Creating EKS NodeGroup %s", ngMetadata.Name) + logger.Debugf("Creating EKS nodegroup %s", ngMetadata.Name) oldMetadata := eksMetadata.NodeGroups[ngPrefix] ngMetadata.CopyMissingFieldsFrom(oldMetadata) @@ -254,14 +254,14 @@ func (provisioner *EKSProvisioner) CreateNodegroups(cluster *model.Cluster) erro err := provisioner.prepareLaunchTemplate(cluster, ngPrefix, ngMetadata, logger) if err != nil { - logger.WithError(err).Error("failed to ensure launch template") + logger.WithError(err).Error("Failed to prepare launch template") errOccurred = true return } _, err = provisioner.awsClient.EnsureEKSNodeGroup(cluster, ngPrefix) if err != nil { - logger.WithError(err).Errorf("failed to create EKS NodeGroup %s", ngMetadata.Name) + logger.WithError(err).Errorf("Failed to create EKS nodegroup %s", ngMetadata.Name) errOccurred = true return } @@ -271,12 +271,12 @@ func (provisioner *EKSProvisioner) CreateNodegroups(cluster *model.Cluster) erro wg.Wait() if errOccurred { - return errors.New("failed to create one of the EKS NodeGroups") + return errors.New("failed to create one of the EKS nodegroups") } err := provisioner.clusterUpdateStore.UpdateCluster(cluster) if err != nil { - return errors.Wrap(err, "failed to store cluster") + return errors.Wrap(err, "failed to update cluster metadata after creating nodegroups") } return nil @@ -302,11 +302,11 @@ func (provisioner *EKSProvisioner) CheckNodegroupsCreated(cluster *model.Cluster defer wg.Done() wait := 300 - logger.Infof("Waiting up to %d seconds for EKS NodeGroup %s to become active...", wait, ngMetadata.Name) + logger.Infof("Waiting up to %d seconds for EKS nodegroup %s to become active...", wait, ngMetadata.Name) nodeGroup, err := provisioner.awsClient.WaitForActiveEKSNodeGroup(eksMetadata.Name, ngMetadata.Name, wait) if err != nil { - logger.WithError(err).Errorf("failed to wait for EKS NodeGroup %s to become active", ngMetadata.Name) + logger.WithError(err).Errorf("Failed to wait for EKS nodegroup %s to become active", ngMetadata.Name) errOccurred = true return } @@ -325,15 +325,15 @@ func (provisioner *EKSProvisioner) CheckNodegroupsCreated(cluster *model.Cluster wg.Wait() - logger.Debugf("All EKS NodeGroups are active") + logger.Debugf("All EKS nodegroups are active") if errOccurred { - return false, errors.New("one of the EKS NodeGroups failed to become active") + return false, errors.New("one of the EKS nodegroups failed to become active") } err := provisioner.awsClient.InstallEKSAddons(cluster) if err != nil { - return false, errors.Wrap(err, "failed to install EKS EBS Addon") + return false, errors.Wrap(err, "failed to install an EKS addon") } eksMetadata.NodeRoleARN = changeRequest.NodeRoleARN @@ -343,7 +343,7 @@ func (provisioner *EKSProvisioner) CheckNodegroupsCreated(cluster *model.Cluster err = provisioner.clusterUpdateStore.UpdateCluster(cluster) if err != nil { - return false, errors.Wrap(err, "failed to store cluster") + return false, errors.Wrap(err, "failed to save updated cluster metadata") } return true, nil @@ -355,12 +355,12 @@ func (provisioner *EKSProvisioner) DeleteNodegroups(cluster *model.Cluster) erro eksMetadata := cluster.ProvisionerMetadataEKS if eksMetadata == nil { - return errors.New("EKS Metadata not set when deleting EKS NodeGroup") + return errors.New("metadata not found to delete EKS nodegroups") } changeRequest := eksMetadata.ChangeRequest if changeRequest == nil || changeRequest.NodeGroups == nil { - return errors.New("nodegroup change request not set when deleting EKS NodeGroup") + return errors.New("metadata ChangeRequest not found to delete EKS nodegroups") } nodeGroups := changeRequest.NodeGroups @@ -375,16 +375,16 @@ func (provisioner *EKSProvisioner) DeleteNodegroups(cluster *model.Cluster) erro err := provisioner.awsClient.EnsureEKSNodeGroupDeleted(eksMetadata.Name, ngMetadata.Name) if err != nil { - logger.WithError(err).Errorf("failed to delete EKS NodeGroup %s", ngMetadata.Name) + logger.WithError(err).Errorf("Failed to delete EKS nodedroup %s", ngMetadata.Name) errOccurred = true return } wait := 600 - logger.Infof("Waiting up to %d seconds for NodeGroup %s to be deleted...", wait, ngMetadata.Name) + logger.Infof("Waiting up to %d seconds for EKS nodegroup %s to be deleted...", wait, ngMetadata.Name) err = provisioner.awsClient.WaitForEKSNodeGroupToBeDeleted(eksMetadata.Name, ngMetadata.Name, wait) if err != nil { - logger.WithError(err).Errorf("failed to delete EKS NodeGroup %s", ngMetadata.Name) + logger.WithError(err).Errorf("Failed to delete EKS nodegroup %s", ngMetadata.Name) errOccurred = true return } @@ -392,19 +392,19 @@ func (provisioner *EKSProvisioner) DeleteNodegroups(cluster *model.Cluster) erro launchTemplateName := fmt.Sprintf("%s-%s", eksMetadata.Name, ngPrefix) err = provisioner.awsClient.DeleteLaunchTemplate(launchTemplateName) if err != nil { - logger.WithError(err).Errorf("failed to delete EKS LaunchTemplate %s", launchTemplateName) + logger.WithError(err).Errorf("Failed to delete launch template %s", launchTemplateName) errOccurred = true return } - logger.Debugf("Successfully deleted EKS NodeGroup %s", ngMetadata.Name) + logger.Debugf("Successfully deleted EKS nodegroup %s", ngMetadata.Name) }(ng, meta) } wg.Wait() if errOccurred { - return errors.New("failed to delete one of the nodegroups") + return errors.New("failed to delete one of the EKS nodegroups") } for ng := range nodeGroups { @@ -420,7 +420,7 @@ func (provisioner *EKSProvisioner) ProvisionCluster(cluster *model.Cluster) erro eksMetadata := cluster.ProvisionerMetadataEKS if eksMetadata == nil { - return errors.New("expected EKS metadata not to be nil when using EKS Provisioner") + return errors.New("metadata not found to provision EKS cluster") } kubeConfigPath, err := provisioner.getKubeConfigPath(cluster) @@ -440,17 +440,17 @@ func (provisioner *EKSProvisioner) UpgradeCluster(cluster *model.Cluster) error err := eksMetadata.ValidateChangeRequest() if err != nil { - return errors.Wrap(err, "eks Metadata ChangeRequest failed validation") + return errors.Wrap(err, "failed to validate ChangeRequest for EKS cluster") } if changeRequest.AMI != "" && changeRequest.AMI != "latest" { var isAMIValid bool isAMIValid, err = provisioner.awsClient.IsValidAMI(eksMetadata.ChangeRequest.AMI, logger) if err != nil { - return errors.Wrapf(err, "error checking the AWS ami image %s", eksMetadata.ChangeRequest.AMI) + return errors.Wrapf(err, "failed to validate AMI %s", eksMetadata.ChangeRequest.AMI) } if !isAMIValid { - return errors.Errorf("invalid AWS ami image %s", eksMetadata.ChangeRequest.AMI) + return errors.Errorf("provided AMI %s is not valid", eksMetadata.ChangeRequest.AMI) } } @@ -463,7 +463,7 @@ func (provisioner *EKSProvisioner) UpgradeCluster(cluster *model.Cluster) error wg.Add(1) go func(ngPrefix string, ngMetadata model.NodeGroupMetadata) { defer wg.Done() - logger.Debugf("Migrating EKS NodeGroup for %s", ngPrefix) + logger.Debugf("Migrating EKS nodegroup for %s", ngPrefix) oldMetadata := eksMetadata.NodeGroups[ngPrefix] ngMetadata.CopyMissingFieldsFrom(oldMetadata) @@ -471,14 +471,14 @@ func (provisioner *EKSProvisioner) UpgradeCluster(cluster *model.Cluster) error err = provisioner.prepareLaunchTemplate(cluster, ngPrefix, ngMetadata, logger) if err != nil { - logger.WithError(err).Error("failed to ensure launch template") + logger.WithError(err).Error("Failed to prepare launch template") errOccurred = true return } err = provisioner.awsClient.EnsureEKSNodeGroupMigrated(cluster, ngPrefix) if err != nil { - logger.WithError(err).Errorf("failed to migrate EKS NodeGroup for %s", ngPrefix) + logger.WithError(err).Errorf("Failed to migrate EKS nodegroup for %s", ngPrefix) errOccurred = true return } @@ -486,14 +486,14 @@ func (provisioner *EKSProvisioner) UpgradeCluster(cluster *model.Cluster) error oldMetadata.Name = ngMetadata.Name eksMetadata.NodeGroups[ngPrefix] = oldMetadata - logger.Debugf("Successfully migrated EKS NodeGroup for %s", ngPrefix) + logger.Debugf("Successfully migrated EKS nodegroup for %s", ngPrefix) }(ng, meta) } wg.Wait() if errOccurred { - return errors.New("failed to migrate one of the EKS NodeGroups") + return errors.New("failed to migrate one of the EKS nodegroups") } if changeRequest.AMI != "" { @@ -505,7 +505,7 @@ func (provisioner *EKSProvisioner) UpgradeCluster(cluster *model.Cluster) error err = provisioner.clusterUpdateStore.UpdateCluster(cluster) if err != nil { - return errors.Wrap(err, "failed to store cluster") + return errors.Wrap(err, "failed to update cluster metadata") } } @@ -528,7 +528,7 @@ func (provisioner *EKSProvisioner) UpgradeCluster(cluster *model.Cluster) error err = provisioner.clusterUpdateStore.UpdateCluster(cluster) if err != nil { - return errors.Wrap(err, "failed to store cluster") + return errors.Wrap(err, "failed to update cluster metadata") } } @@ -549,7 +549,7 @@ func (provisioner *EKSProvisioner) ResizeCluster(cluster *model.Cluster) error { err := eksMetadata.ValidateChangeRequest() if err != nil { - return errors.Wrap(err, "eks Metadata ChangeRequest failed validation") + return errors.Wrap(err, "failed to validate ChangeRequest for EKS cluster") } var wg sync.WaitGroup @@ -559,7 +559,7 @@ func (provisioner *EKSProvisioner) ResizeCluster(cluster *model.Cluster) error { wg.Add(1) go func(ngPrefix string, ngMetadata model.NodeGroupMetadata) { defer wg.Done() - logger.Debugf("Migrating EKS NodeGroup for %s", ngPrefix) + logger.Debugf("Migrating EKS nodegroup for %s", ngPrefix) oldMetadata := eksMetadata.NodeGroups[ngPrefix] ngMetadata.CopyMissingFieldsFrom(oldMetadata) @@ -567,45 +567,45 @@ func (provisioner *EKSProvisioner) ResizeCluster(cluster *model.Cluster) error { oldEKSNodeGroup, err2 := provisioner.awsClient.GetActiveEKSNodeGroup(eksMetadata.Name, oldMetadata.Name) if err2 != nil { - logger.WithError(err2).Errorf("failed to get EKS NodeGroup for %s", ngPrefix) + logger.WithError(err2).Errorf("Failed to get the existing EKS nodegroup for %s", ngPrefix) errOccurred = true return } if oldEKSNodeGroup == nil || oldEKSNodeGroup.Status != eksTypes.NodegroupStatusActive { - logger.Debugf("No active EKS NodeGroup found for %s", ngPrefix) + logger.Debugf("No active EKS nodegroup found for %s", ngPrefix) return } if oldMetadata.InstanceType == ngMetadata.InstanceType && oldMetadata.MinCount == ngMetadata.MinCount && oldMetadata.MaxCount == ngMetadata.MaxCount { - logger.Debugf("No change in EKS NodeGroup for %s", ngPrefix) + logger.Debugf("No change in EKS nodegroup for %s", ngPrefix) } err = provisioner.prepareLaunchTemplate(cluster, ngPrefix, ngMetadata, logger) if err != nil { - logger.WithError(err).Error("failed to ensure launch template") + logger.WithError(err).Error("Failed to prepare launch template") errOccurred = true return } err = provisioner.awsClient.EnsureEKSNodeGroupMigrated(cluster, ngPrefix) if err != nil { - logger.WithError(err).Errorf("failed to migrate EKS NodeGroup for %s", ngPrefix) + logger.WithError(err).Errorf("Failed to migrate EKS nodegroup for %s", ngPrefix) errOccurred = true return } nodeGroup, err2 := provisioner.awsClient.GetActiveEKSNodeGroup(eksMetadata.Name, ngMetadata.Name) if err2 != nil { - logger.WithError(err2).Errorf("failed to get EKS NodeGroup %s", ngMetadata.Name) + logger.WithError(err2).Errorf("Failed to get EKS nodegroup %s", ngMetadata.Name) errOccurred = true return } if nodeGroup == nil { - logger.WithError(err2).Errorf("EKS NodeGroup %s not found", ngMetadata.Name) + logger.WithError(err2).Errorf("EKS nodegroup %s not found", ngMetadata.Name) errOccurred = true return } @@ -617,19 +617,19 @@ func (provisioner *EKSProvisioner) ResizeCluster(cluster *model.Cluster) error { oldNodeGroup.MaxCount = int64(ptr.ToInt32(nodeGroup.ScalingConfig.MaxSize)) eksMetadata.NodeGroups[ngPrefix] = oldNodeGroup - logger.Debugf("Successfully migrated EKS NodeGroup for %s", ngPrefix) + logger.Debugf("Successfully migrated EKS nodegroup for %s", ngPrefix) }(ng, meta) } wg.Wait() if errOccurred { - return errors.New("failed to migrate one of the nodegroups") + return errors.New("failed to migrate one of the EKS nodegroups") } err = provisioner.clusterUpdateStore.UpdateCluster(cluster) if err != nil { - return errors.Wrap(err, "failed to store cluster") + return errors.Wrap(err, "failed to update cluster metadata") } return nil @@ -673,16 +673,16 @@ func (provisioner *EKSProvisioner) cleanupCluster(cluster *model.Cluster) error err = provisioner.awsClient.EnsureEKSNodeGroupDeleted(eksMetadata.Name, ngMetadata.Name) if err != nil { - logger.WithError(err).Errorf("failed to delete EKS NodeGroup %s", ngMetadata.Name) + logger.WithError(err).Errorf("Failed to delete EKS nodegroup %s", ngMetadata.Name) errOccurred = true return } wait := 600 - logger.Infof("Waiting up to %d seconds for NodeGroup %s to be deleted...", wait, ngMetadata.Name) + logger.Infof("Waiting up to %d seconds for nodegroup %s to be deleted...", wait, ngMetadata.Name) err = provisioner.awsClient.WaitForEKSNodeGroupToBeDeleted(eksMetadata.Name, ngMetadata.Name, wait) if err != nil { - logger.WithError(err).Errorf("failed to delete EKS NodeGroup %s", ngMetadata.Name) + logger.WithError(err).Errorf("Failed to delete EKS nodegroup %s", ngMetadata.Name) errOccurred = true return } @@ -690,7 +690,7 @@ func (provisioner *EKSProvisioner) cleanupCluster(cluster *model.Cluster) error launchTemplateName := fmt.Sprintf("%s-%s", eksMetadata.Name, ngPrefix) err = provisioner.awsClient.DeleteLaunchTemplate(launchTemplateName) if err != nil { - logger.WithError(err).Errorf("failed to delete EKS LaunchTemplate %s", launchTemplateName) + logger.WithError(err).Errorf("Failed to delete launch template %s", launchTemplateName) errOccurred = true return } @@ -702,7 +702,7 @@ func (provisioner *EKSProvisioner) cleanupCluster(cluster *model.Cluster) error wg.Wait() if errOccurred { - return errors.New("failed to delete one of the nodegroups") + return errors.New("failed to delete one of the EKS nodegroups") } err = provisioner.awsClient.EnsureEKSClusterDeleted(eksMetadata.Name) @@ -747,11 +747,11 @@ func (provisioner *EKSProvisioner) setMissingChangeRequest(eksMetadata *model.EK func (provisioner *EKSProvisioner) DeleteCluster(cluster *model.Cluster) (bool, error) { logger := provisioner.logger.WithField("cluster", cluster.ID) - logger.Info("Deleting cluster") + logger.Info("Deleting EKS cluster") eksMetadata := cluster.ProvisionerMetadataEKS if eksMetadata == nil { - return false, errors.New("expected EKS metadata not to be nil when using EKS Provisioner") + return false, errors.New("metadata not found to delete EKS cluster") } eksCluster, err := provisioner.awsClient.GetActiveEKSCluster(eksMetadata.Name) @@ -794,7 +794,7 @@ func (provisioner *EKSProvisioner) getKubeConfigPath(cluster *model.Cluster) (st return "", errors.Wrap(err, "failed to get EKS cluster") } if eksCluster == nil { - return "", errors.New("EKS cluster not ready") + return "", errors.Errorf("the EKS cluster %s does not exist", clusterName) } kubeconfig, err := newEKSKubeConfig(eksCluster, provisioner.awsClient) @@ -804,7 +804,7 @@ func (provisioner *EKSProvisioner) getKubeConfigPath(cluster *model.Cluster) (st kubeconfigFile, err := os.CreateTemp("", clusterName) if err != nil { - return "", errors.Wrap(err, "failed to create kubeconfig tempfile") + return "", errors.Wrap(err, "failed to create temp file for kubeconfig") } defer kubeconfigFile.Close() @@ -823,7 +823,7 @@ func (provisioner *EKSProvisioner) getKubeConfigPath(cluster *model.Cluster) (st func (provisioner *EKSProvisioner) getKubeClient(cluster *model.Cluster) (*k8s.KubeClient, error) { configLocation, err := provisioner.getKubeConfigPath(cluster) if err != nil { - return nil, errors.Wrap(err, "failed to get kube config") + return nil, errors.Wrap(err, "failed to get kubeconfig") } var k8sClient *k8s.KubeClient diff --git a/internal/supervisor/cluster.go b/internal/supervisor/cluster.go index ff8baf073..c42c79cfe 100644 --- a/internal/supervisor/cluster.go +++ b/internal/supervisor/cluster.go @@ -130,7 +130,7 @@ func (s *ClusterSupervisor) Supervise(cluster *model.Cluster) { cluster, err = s.store.GetCluster(cluster.ID) if err != nil { - logger.WithError(err).Warnf("failed to get cluster and thus persist state %s", newState) + logger.WithError(err).Warnf("Failed to get cluster and thus persist state %s", newState) return } @@ -142,7 +142,7 @@ func (s *ClusterSupervisor) Supervise(cluster *model.Cluster) { cluster.State = newState err = s.store.UpdateCluster(cluster) if err != nil { - logger.WithError(err).Warnf("failed to set cluster state to %s", newState) + logger.WithError(err).Warnf("Failed to set cluster state to %s", newState) return } @@ -226,10 +226,10 @@ func (s *ClusterSupervisor) upgradeCluster(cluster *model.Cluster, logger log.Fi err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).UpgradeCluster(cluster) if err != nil { logger.WithError(err).Error("Failed to upgrade cluster") - logger.Info("Updating cluster store with latest cluster data") + logger.Info("Updating cluster metadata to reflect upgrade failure") err = s.store.UpdateCluster(cluster) if err != nil { - logger.WithError(err).Error("Failed to save updated cluster metadata") + logger.WithError(err).Error("Failed to update cluster metadata to reflect upgrade failure") return model.ClusterStateRefreshMetadata } return model.ClusterStateUpgradeFailed @@ -274,7 +274,7 @@ func (s *ClusterSupervisor) deleteNodegroups(cluster *model.Cluster, logger log. return model.ClusterStateNodegroupsDeletionFailed } - logger.Info("Finished deleting nodegroup") + logger.Info("Finished deleting nodegroups") return s.refreshClusterMetadata(cluster, logger) } @@ -282,13 +282,13 @@ func (s *ClusterSupervisor) refreshClusterMetadata(cluster *model.Cluster, logge err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).RefreshClusterMetadata(cluster) if err != nil { - logger.WithError(err).Error("Failed to refresh cluster") + logger.WithError(err).Error("Failed to refresh cluster metadata") return model.ClusterStateRefreshMetadata } err = s.store.UpdateCluster(cluster) if err != nil { - logger.WithError(err).Error("Failed to save updated cluster metadata") + logger.WithError(err).Error("Failed to update cluster metadata in store") return model.ClusterStateRefreshMetadata } @@ -308,7 +308,7 @@ func (s *ClusterSupervisor) deleteCluster(cluster *model.Cluster, logger log.Fie err = s.store.DeleteCluster(cluster.ID) if err != nil { - logger.WithError(err).Error("Failed to record updated cluster after deletion") + logger.WithError(err).Error("Failed to delete cluster from store") return model.ClusterStateDeletionFailed } @@ -323,13 +323,13 @@ func (s *ClusterSupervisor) checkClusterCreated(cluster *model.Cluster, logger l return model.ClusterStateCreationFailed } if !ready { - logger.Debug("Cluster not yet ready") + logger.Debug("Cluster is not yet ready") return model.ClusterStateCreationInProgress } err = s.provisioner.GetClusterProvisioner(cluster.Provisioner).CreateNodegroups(cluster) if err != nil { - logger.WithError(err).Error("Failed to create cluster nodes") + logger.WithError(err).Error("Failed to create cluster nodegroups") return model.ClusterStateCreationFailed } @@ -340,11 +340,11 @@ func (s *ClusterSupervisor) checkNodesCreated(cluster *model.Cluster, logger log ready, err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).CheckNodegroupsCreated(cluster) if err != nil { - logger.WithError(err).Error("Failed to check if node creation finished") + logger.WithError(err).Error("Failed to check if nodegroups creation finished") return model.ClusterStateCreationFailed } if !ready { - logger.Debug("Cluster nodes are not ready yet") + logger.Debug("Cluster nodegroups are not ready yet") return model.ClusterStateWaitingForNodes } diff --git a/internal/tools/aws/ec2.go b/internal/tools/aws/ec2.go index 012947086..f9a5982fb 100644 --- a/internal/tools/aws/ec2.go +++ b/internal/tools/aws/ec2.go @@ -179,7 +179,7 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { eksCluster, err := a.getEKSCluster(data.ClusterName) if err != nil { - return errors.Wrap(err, "failed to get eks cluster") + return errors.Wrap(err, "failed to get EKS Cluster") } userData := getLaunchTemplateUserData(eksCluster, data) @@ -205,14 +205,14 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { }) if err != nil { if IsErrorCode(err, "InvalidLaunchTemplateName.AlreadyExistsException") { - a.logger.Debugf("Launch template %s already exists", data.Name) + a.logger.Debugf("LaunchTemplate %s already exists", data.Name) return nil } - return errors.Wrap(err, "failed to create eks launch template") + return errors.Wrap(err, "failed to create EKS LaunchTemplate") } if launchTemplate == nil || launchTemplate.LaunchTemplate == nil { - return errors.New("failed to create eks launch template") + return errors.New("failed to create EKS LaunchTemplate") } return nil @@ -225,7 +225,7 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { eksCluster, err := a.getEKSCluster(data.ClusterName) if err != nil { - return errors.Wrap(err, "failed to get eks cluster") + return errors.Wrap(err, "failed to get ESK Cluster") } userData := getLaunchTemplateUserData(eksCluster, data) @@ -253,11 +253,11 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { if IsErrorCode(err, "InvalidLaunchTemplateName.NotFoundException") { return a.CreateLaunchTemplate(data) } - return errors.Wrap(err, "failed to create eks launch template version") + return errors.Wrap(err, "failed to create EKS LaunchTemplate version") } if launchTemplate == nil || launchTemplate.LaunchTemplateVersion == nil { - return errors.New("failed to create eks launch template version") + return errors.New("failed to create ESK LaunchTemplate version") } return nil @@ -270,7 +270,7 @@ func (a *Client) DeleteLaunchTemplate(launchTemplateName string) error { } if launchTemplate == nil { - a.logger.Debugf("launch template %s not found, assuming deleted", launchTemplateName) + a.logger.Debugf("LaunchTemplate %s not found, assuming deleted", launchTemplateName) return nil } @@ -282,7 +282,7 @@ func (a *Client) DeleteLaunchTemplate(launchTemplateName string) error { a.logger.Debugf("launch template %s not found, assuming deleted", launchTemplateName) return nil } - return errors.Wrap(err, "failed to delete eks launch template") + return errors.Wrap(err, "failed to delete EKS LaunchTemplate") } return nil diff --git a/internal/tools/aws/eks.go b/internal/tools/aws/eks.go index e2eed50b6..d826d8798 100644 --- a/internal/tools/aws/eks.go +++ b/internal/tools/aws/eks.go @@ -26,7 +26,6 @@ func (c *Client) createEKSCluster(cluster *model.Cluster, resources ClusterResou // TODO: we do not expect to query that many subnets but for safety // we can check the NextToken. subnetsOut, err := c.Service().ec2.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{ - // TODO: is it public/private SubnetIds: resources.PrivateSubnetIDs, }) if err != nil { @@ -60,7 +59,6 @@ func (c *Client) createEKSCluster(cluster *model.Cluster, resources ClusterResou } eksMetadata := cluster.ProvisionerMetadataEKS - // TODO: we can allow further parametrization in the future input := eks.CreateClusterInput{ Name: aws.String(eksMetadata.Name), ResourcesVpcConfig: &vpcConfig, @@ -72,7 +70,7 @@ func (c *Client) createEKSCluster(cluster *model.Cluster, resources ClusterResou out, err := c.Service().eks.CreateCluster(ctx, &input) if err != nil { - return nil, errors.Wrap(err, "failed to create EKS cluster") + return nil, errors.Wrap(err, "failed to create EKS Cluster") } return out.Cluster, nil @@ -85,7 +83,7 @@ func (a *Client) getEKSCluster(clusterName string) (*eksTypes.Cluster, error) { }) if err != nil { if !IsErrorResourceNotFound(err) { - return nil, errors.Wrap(err, "failed to describe cluster") + return nil, errors.Wrap(err, "failed to describe EKS Cluster") } } @@ -164,11 +162,11 @@ func (c *Client) EnsureEKSClusterUpdated(cluster *model.Cluster) (*eksTypes.Upda } if eksCluster == nil { - return nil, errors.Errorf("cluster %s does not exist", clusterName) + return nil, errors.Errorf("requested EKS Cluster %s is not found", clusterName) } if eksCluster.Status != eksTypes.ClusterStatusActive { - return nil, errors.Errorf("cluster %s is not active", clusterName) + return nil, errors.Errorf("requested EKS Cluster %s is not active", clusterName) } eksMetadata := cluster.ProvisionerMetadataEKS @@ -182,7 +180,7 @@ func (c *Client) EnsureEKSClusterUpdated(cluster *model.Cluster) (*eksTypes.Upda }) if err != nil { - return nil, errors.Wrap(err, "failed to update EKS cluster version") + return nil, errors.Wrap(err, "failed to update EKS Cluster version") } return output.Update, nil @@ -193,7 +191,7 @@ func (a *Client) createEKSNodeGroup(cluster *model.Cluster, ngPrefix string) (*e eksMetadata := cluster.ProvisionerMetadataEKS changeRequest := eksMetadata.ChangeRequest if changeRequest == nil { - return nil, errors.New("change request is nil") + return nil, errors.New("metadata ChangeRequest is not set") } clusterResource, err := a.GetVpcResourcesByVpcID(changeRequest.VPC, a.logger) @@ -274,21 +272,21 @@ func (a *Client) createEKSNodeGroup(cluster *model.Cluster, ngPrefix string) (*e out, err := a.Service().eks.CreateNodegroup(context.TODO(), &nodeGroupReq) if err != nil { - return nil, errors.Wrap(err, "failed to create EKS NodeGroup") + return nil, errors.Wrap(err, "failed to create EKS Nodegroup") } return out.Nodegroup, nil } -func (c *Client) getEKSNodeGroup(clusterName, workerName string) (*eksTypes.Nodegroup, error) { +func (c *Client) getEKSNodeGroup(clusterName, nodegroupName string) (*eksTypes.Nodegroup, error) { output, err := c.Service().eks.DescribeNodegroup(context.TODO(), &eks.DescribeNodegroupInput{ ClusterName: aws.String(clusterName), - NodegroupName: aws.String(workerName), + NodegroupName: aws.String(nodegroupName), }) if err != nil { if !IsErrorResourceNotFound(err) { - return nil, errors.Wrap(err, "failed to describe EKS NodeGroup") + return nil, errors.Wrap(err, "failed to describe EKS Nodegroup") } } @@ -309,12 +307,12 @@ func (c *Client) EnsureEKSNodeGroup(cluster *model.Cluster, ngPrefix string) (*e ngChangeRequest, found := changeRequest.NodeGroups[ngPrefix] if !found { - return nil, errors.Errorf("nodegroup %s not found in change request", ngPrefix) + return nil, errors.Errorf("nodegroup metadata for %s not found in ChangeRequest", ngPrefix) } nodeGroup, err := c.getEKSNodeGroup(clusterName, ngChangeRequest.Name) if err != nil { - return nil, errors.Wrapf(err, "failed to get an EKS NodeGroup %s", ngChangeRequest.Name) + return nil, errors.Wrapf(err, "failed to get an EKS Nodegroup %s", ngChangeRequest.Name) } if nodeGroup != nil { @@ -333,7 +331,7 @@ func (c *Client) EnsureEKSNodeGroupMigrated(cluster *model.Cluster, ngPrefix str ngChangeRequest, found := changeRequest.NodeGroups[ngPrefix] if !found { - return errors.Errorf("nodegroup meta for %s not found in change request", ngPrefix) + return errors.Errorf("nodegroup metadata for %s not found in ChangeRequest", ngPrefix) } clusterName := eksMetadata.Name @@ -346,25 +344,25 @@ func (c *Client) EnsureEKSNodeGroupMigrated(cluster *model.Cluster, ngPrefix str _, err := c.createEKSNodeGroup(cluster, ngPrefix) if err != nil { - return errors.Wrapf(err, "failed to create a new EKS NodeGroup %s", ngChangeRequest.Name) + return errors.Wrapf(err, "failed to create a new EKS Nodegroup %s", ngChangeRequest.Name) } wait := 600 // seconds - logger.Infof("Waiting up to %d seconds for EKS NodeGroup %s to become active...", wait, ngChangeRequest.Name) + logger.Infof("Waiting up to %d seconds for EKS Nodegroup %s to become active...", wait, ngChangeRequest.Name) _, err = c.WaitForActiveEKSNodeGroup(eksMetadata.Name, ngChangeRequest.Name, wait) if err != nil { return err } - logger.Debugf("Deleting the old EKS NodeGroup %s", oldNodeGroupName) + logger.Debugf("Deleting the old EKS Nodegroup %s", oldNodeGroupName) err = c.EnsureEKSNodeGroupDeleted(clusterName, oldNodeGroupName) if err != nil { - return errors.Wrapf(err, "failed to delete the old EKS NodeGroup %s", oldNodeGroupName) + return errors.Wrapf(err, "failed to delete the old EKS Nodegroup %s", oldNodeGroupName) } - logger.Infof("Waiting up to %d seconds for EKS NodeGroup %s to be deleted...", wait, oldNodeGroupName) + logger.Infof("Waiting up to %d seconds for EKS Nodegroup %s to be deleted...", wait, oldNodeGroupName) err = c.WaitForEKSNodeGroupToBeDeleted(eksMetadata.Name, oldNodeGroupName, wait) if err != nil { return err @@ -379,7 +377,7 @@ func (a *Client) EnsureEKSClusterDeleted(clusterName string) error { eksCluster, err := a.getEKSCluster(clusterName) if err != nil { - return errors.Wrap(err, "failed to describe EKS cluster") + return errors.Wrap(err, "failed to describe EKS Cluster") } if eksCluster == nil { @@ -392,13 +390,13 @@ func (a *Client) EnsureEKSClusterDeleted(clusterName string) error { } if eksCluster.Status == eksTypes.ClusterStatusFailed { - return errors.New("cluster is in failed state") + return errors.New("requested EKS Cluster is in failed state") } delInput := &eks.DeleteClusterInput{Name: aws.String(clusterName)} _, err = a.Service().eks.DeleteCluster(ctx, delInput) if err != nil { - return errors.Wrap(err, "failed to trigger EKS cluster deletion") + return errors.Wrap(err, "failed to trigger EKS Cluster deletion") } // Cluster just started deletion @@ -406,14 +404,14 @@ func (a *Client) EnsureEKSClusterDeleted(clusterName string) error { } // EnsureEKSNodeGroupDeleted ensures EKS node groups are deleted. -func (a *Client) EnsureEKSNodeGroupDeleted(clusterName, workerName string) error { - if workerName == "" { +func (a *Client) EnsureEKSNodeGroupDeleted(clusterName, nodegroupName string) error { + if nodegroupName == "" { return nil } - nodeGroups, err := a.getEKSNodeGroup(clusterName, workerName) + nodeGroups, err := a.getEKSNodeGroup(clusterName, nodegroupName) if err != nil { - return errors.Wrap(err, "failed to get NodeGroup") + return errors.Wrap(err, "failed to get EKS Nodegroup") } // Node groups deleted, we can return if nodeGroups == nil { @@ -425,7 +423,7 @@ func (a *Client) EnsureEKSNodeGroupDeleted(clusterName, workerName string) error } if nodeGroups.Status == eksTypes.NodegroupStatusDeleteFailed { - return errors.Wrapf(err, "node group deletion failed %q", *nodeGroups.NodegroupName) + return errors.Wrapf(err, "failed to delete EKS Nodegroup %s", nodegroupName) } _, err = a.Service().eks.DeleteNodegroup(context.TODO(), &eks.DeleteNodegroupInput{ @@ -433,7 +431,7 @@ func (a *Client) EnsureEKSNodeGroupDeleted(clusterName, workerName string) error NodegroupName: nodeGroups.NodegroupName, }) if err != nil { - return errors.Wrap(err, "failed to delete NodeGroup") + return errors.Wrap(err, "failed to delete EKS Nodegroup") } return nil @@ -443,7 +441,7 @@ func (a *Client) EnsureEKSNodeGroupDeleted(clusterName, workerName string) error func (c *Client) GetActiveEKSCluster(clusterName string) (*eksTypes.Cluster, error) { cluster, err := c.getEKSCluster(clusterName) if err != nil { - return nil, errors.Wrap(err, "failed to get EKS cluster") + return nil, errors.Wrap(err, "failed to get EKS Cluster") } if cluster == nil { @@ -451,7 +449,7 @@ func (c *Client) GetActiveEKSCluster(clusterName string) (*eksTypes.Cluster, err } if cluster.Status == eksTypes.ClusterStatusFailed { - return nil, errors.New("cluster creation failed") + return nil, errors.New("requested EKS Cluster is in failed state") } if cluster.Status == eksTypes.ClusterStatusActive { @@ -465,7 +463,7 @@ func (c *Client) GetActiveEKSCluster(clusterName string) (*eksTypes.Cluster, err func (c *Client) GetActiveEKSNodeGroup(clusterName, workerName string) (*eksTypes.Nodegroup, error) { nodeGroup, err := c.getEKSNodeGroup(clusterName, workerName) if err != nil { - return nil, errors.Wrap(err, "failed to get NodeGroup") + return nil, errors.Wrap(err, "failed to get EKS Nodegroup") } if nodeGroup == nil { @@ -473,7 +471,7 @@ func (c *Client) GetActiveEKSNodeGroup(clusterName, workerName string) (*eksType } if nodeGroup.Status == eksTypes.NodegroupStatusCreateFailed { - return nil, errors.New("EKS NodeGroup creation failed") + return nil, errors.New("failed to create EKS Nodegroup") } if nodeGroup.Status == eksTypes.NodegroupStatusActive { @@ -493,11 +491,11 @@ func (c *Client) WaitForActiveEKSCluster(clusterName string, timeout int) (*eksT for { select { case <-timeoutTimer.C: - return nil, errors.New("timed out waiting for EKS cluster to become active") + return nil, errors.New("timed out waiting for EKS Cluster to become active") case <-tick.C: eksCluster, err := c.GetActiveEKSCluster(clusterName) if err != nil { - return nil, errors.Wrap(err, "failed to check if EKS cluster is active") + return nil, errors.Wrap(err, "failed to check if EKS Cluster is active") } if eksCluster != nil { return eksCluster, nil @@ -515,11 +513,11 @@ func (c *Client) WaitForActiveEKSNodeGroup(clusterName, nodeGroupName string, ti for { select { case <-timeoutTimer.C: - return nil, errors.New("timed out waiting for EKS NodeGroup to become active") + return nil, errors.New("timed out waiting for EKS Nodegroup to become active") case <-tick.C: nodeGroup, err := c.GetActiveEKSNodeGroup(clusterName, nodeGroupName) if err != nil { - return nil, errors.Wrap(err, "failed to check if EKS NodeGroup is active") + return nil, errors.Wrap(err, "failed to check if EKS Nodegroup is active") } if nodeGroup != nil { return nodeGroup, nil @@ -537,11 +535,11 @@ func (c *Client) WaitForEKSNodeGroupToBeDeleted(clusterName, workerName string, for { select { case <-timeoutTimer.C: - return errors.New("timed out waiting for EKS NodeGroup to be deleted") + return errors.New("timed out waiting for EKS Nodegroup to be deleted") case <-tick.C: nodeGroup, err := c.getEKSNodeGroup(clusterName, workerName) if err != nil { - return errors.Wrap(err, "failed to describe NodeGroup") + return errors.Wrap(err, "failed to describe EKS Nodegroup") } if nodeGroup == nil { return nil @@ -559,11 +557,11 @@ func (c *Client) WaitForEKSClusterToBeDeleted(clusterName string, timeout int) e for { select { case <-timeoutTimer.C: - return errors.New("timed out waiting for EKS cluster to become ready") + return errors.New("timed out waiting for EKS Cluster to become ready") case <-tick.C: eksCluster, err := c.getEKSCluster(clusterName) if err != nil { - return errors.Wrap(err, "failed to describe EKS cluster") + return errors.Wrap(err, "failed to describe EKS Cluster") } if eksCluster == nil { return nil @@ -581,15 +579,15 @@ func (c *Client) WaitForEKSClusterUpdateToBeCompleted(clusterName, updateID stri for { select { case <-timeoutTimer.C: - return errors.New("timed out waiting for EKS cluster update to be completed") + return errors.New("timed out waiting for EKS Cluster update to be completed") case <-tick.C: updateStatus, err := c.getEKSClusterUpdateStatus(clusterName, updateID) if err != nil { - return errors.Wrap(err, "failed to describe EKS cluster") + return errors.Wrap(err, "failed to describe EKS Cluster") } if updateStatus == eksTypes.UpdateStatusFailed { - return errors.New("EKS cluster update failed") + return errors.New("failed to update EKS Cluster") } if updateStatus == eksTypes.UpdateStatusSuccessful { @@ -605,7 +603,7 @@ func (c *Client) getEKSClusterUpdateStatus(clusterName, updateID string) (eksTyp UpdateId: ptr.String(updateID), }) if err != nil { - return "", errors.Wrap(err, "failed to describe EKS cluster update") + return "", errors.Wrap(err, "failed to describe update for EKS Cluster") } return output.Update.Status, nil diff --git a/model/cluster_request.go b/model/cluster_request.go index 966d2284d..a7e3113ea 100644 --- a/model/cluster_request.go +++ b/model/cluster_request.go @@ -409,10 +409,10 @@ func (request *CreateNodegroupsRequest) SetDefaults() { func (request *CreateNodegroupsRequest) Validate() error { for ng, meta := range request.Nodegroups { if meta.MinCount < 1 { - return errors.Errorf("nodegroup %s min count has to be 1 or greater", ng) + return errors.Errorf("minimum node count has to be 1 or greater for nodegroup %s", ng) } if meta.MaxCount < meta.MinCount { - return errors.Errorf("nodegroup %s max count (%d) can't be less than min count (%d)", ng, meta.MaxCount, meta.MinCount) + return errors.Errorf("maximum node count (%d) can't be less than minimum node count (%d) for nodegroup %s", meta.MaxCount, meta.MinCount, ng) } } @@ -421,7 +421,7 @@ func (request *CreateNodegroupsRequest) Validate() error { continue } if _, f := request.Nodegroups[ng]; !f { - return errors.Errorf("invalid nodegroup %s to use public subnets", ng) + return errors.Errorf("nodegroup %s not found to use public subnet", ng) } } @@ -430,7 +430,7 @@ func (request *CreateNodegroupsRequest) Validate() error { continue } if _, f := request.Nodegroups[ng]; !f { - return errors.Errorf("invalid nodegroup %s to use security group", ng) + return errors.Errorf("nodegroup %s not found to use security group", ng) } } diff --git a/model/eks_metadata.go b/model/eks_metadata.go index 6b606adf9..946be465e 100644 --- a/model/eks_metadata.go +++ b/model/eks_metadata.go @@ -163,7 +163,7 @@ func (em *EKSMetadata) ValidateClusterSizePatch(patchRequest *PatchClusterSizeRe nodeGroups := patchRequest.NodeGroups if len(em.NodeGroups) == 0 { - return errors.New("no nodegroups available to resize") + return errors.New("no nodegroups are available to resize") } if len(nodeGroups) == 0 { @@ -183,7 +183,7 @@ func (em *EKSMetadata) ValidateClusterSizePatch(patchRequest *PatchClusterSizeRe if patchRequest.NodeMinCount != nil && patchRequest.NodeMaxCount != nil { if *patchRequest.NodeMinCount > *patchRequest.NodeMaxCount { - return errors.New("min node count cannot be greater than max node count") + return errors.New("minimum node count cannot be greater than maximum node count") } return nil } @@ -193,7 +193,7 @@ func (em *EKSMetadata) ValidateClusterSizePatch(patchRequest *PatchClusterSizeRe ng := em.NodeGroups[ngToResize] nodeMaxCount := ng.MaxCount if *patchRequest.NodeMinCount > nodeMaxCount { - return errors.New("resize patch would set min node count higher than max node count") + return errors.New("resize patch would set minimum node count higher than maximum node count") } } } @@ -203,7 +203,7 @@ func (em *EKSMetadata) ValidateClusterSizePatch(patchRequest *PatchClusterSizeRe ng := em.NodeGroups[ngToResize] nodeMinCount := ng.MinCount if *patchRequest.NodeMaxCount < nodeMinCount { - return errors.New("resize patch would set max node count lower than min node count") + return errors.New("resize patch would set maximum node count lower than minimum node count") } } } @@ -257,7 +257,7 @@ func (em *EKSMetadata) ApplyClusterSizePatch(patchRequest *PatchClusterSizeReque func (em *EKSMetadata) ValidateChangeRequest() error { changeRequest := em.ChangeRequest if changeRequest == nil { - return errors.New("the EKS Metadata ChangeRequest is nil") + return errors.New("metadata ChangeRequest is nil") } changeAllowed := false @@ -277,7 +277,7 @@ func (em *EKSMetadata) ValidateChangeRequest() error { } if !changeAllowed { - return errors.New("the EKS Metadata ChangeRequest has no change values set") + return errors.New("metadata ChangeRequest has no changes to apply") } return nil From f376ad7765fd26c7a55fe347b49c90ee50bbb595 Mon Sep 17 00:00:00 2001 From: Mir Shahriar Sabuj Date: Tue, 9 May 2023 19:44:06 +0600 Subject: [PATCH 20/20] updated --- clusterdictionary/size.go | 4 +- internal/api/cluster.go | 2 +- internal/mocks/aws-tools/client.go | 72 +++++++-------- internal/provisioner/eks_provisioner.go | 27 +++--- internal/supervisor/cluster.go | 10 +- internal/supervisor/installation_test.go | 12 +-- internal/tools/aws/client.go | 12 +-- internal/tools/aws/ec2.go | 18 ++-- internal/tools/aws/eks.go | 112 +++++++++++------------ 9 files changed, 135 insertions(+), 134 deletions(-) diff --git a/clusterdictionary/size.go b/clusterdictionary/size.go index 865c9cb23..22b9dbf18 100644 --- a/clusterdictionary/size.go +++ b/clusterdictionary/size.go @@ -184,7 +184,7 @@ func AddToCreateClusterRequest(sizes map[string]string, request *model.CreateClu for ng, ngSize := range sizes { ngType, minCount, maxCount, err := processCustomSize(ngSize) if err != nil { - return errors.Wrapf(err, "invalid nodegroup size for %s", ng) + return err } request.AdditionalNodeGroups[ng] = model.NodeGroupMetadata{ @@ -211,7 +211,7 @@ func AddToCreateNodegroupsRequest(sizes map[string]string, request *model.Create for ng, ngSize := range sizes { ngType, minCount, maxCount, err := processCustomSize(ngSize) if err != nil { - return errors.Wrapf(err, "invalid nodegroup size for %s", ng) + return err } request.Nodegroups[ng] = model.NodeGroupMetadata{ diff --git a/internal/api/cluster.go b/internal/api/cluster.go index 35161e2bd..232fcef84 100644 --- a/internal/api/cluster.go +++ b/internal/api/cluster.go @@ -448,7 +448,7 @@ func handleCreateNodegroups(c *Context, w http.ResponseWriter, r *http.Request) clusterDTO, status, unlockOnce := getClusterForTransition(c, clusterID, newState) if status != 0 { - c.Logger.Debug("Cluster is not in a valid state for nodegroup creation") + c.Logger.Debug("Cluster is not in a valid state for nodegroups creation") w.WriteHeader(status) return } diff --git a/internal/mocks/aws-tools/client.go b/internal/mocks/aws-tools/client.go index cdd865ba2..175960e46 100644 --- a/internal/mocks/aws-tools/client.go +++ b/internal/mocks/aws-tools/client.go @@ -446,33 +446,33 @@ func (mr *MockAWSMockRecorder) EnsureEKSClusterUpdated(cluster interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureEKSClusterUpdated", reflect.TypeOf((*MockAWS)(nil).EnsureEKSClusterUpdated), cluster) } -// EnsureEKSNodeGroup mocks base method -func (m *MockAWS) EnsureEKSNodeGroup(cluster *model.Cluster, nodeGroupPrefix string) (*types.Nodegroup, error) { +// EnsureEKSNodegroup mocks base method +func (m *MockAWS) EnsureEKSNodegroup(cluster *model.Cluster, nodeGroupPrefix string) (*types.Nodegroup, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EnsureEKSNodeGroup", cluster, nodeGroupPrefix) + ret := m.ctrl.Call(m, "EnsureEKSNodegroup", cluster, nodeGroupPrefix) ret0, _ := ret[0].(*types.Nodegroup) ret1, _ := ret[1].(error) return ret0, ret1 } -// EnsureEKSNodeGroup indicates an expected call of EnsureEKSNodeGroup -func (mr *MockAWSMockRecorder) EnsureEKSNodeGroup(cluster, nodeGroupPrefix interface{}) *gomock.Call { +// EnsureEKSNodegroup indicates an expected call of EnsureEKSNodegroup +func (mr *MockAWSMockRecorder) EnsureEKSNodegroup(cluster, nodeGroupPrefix interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureEKSNodeGroup", reflect.TypeOf((*MockAWS)(nil).EnsureEKSNodeGroup), cluster, nodeGroupPrefix) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureEKSNodegroup", reflect.TypeOf((*MockAWS)(nil).EnsureEKSNodegroup), cluster, nodeGroupPrefix) } -// EnsureEKSNodeGroupMigrated mocks base method -func (m *MockAWS) EnsureEKSNodeGroupMigrated(cluster *model.Cluster, nodeGroupPrefix string) error { +// EnsureEKSNodegroupMigrated mocks base method +func (m *MockAWS) EnsureEKSNodegroupMigrated(cluster *model.Cluster, nodeGroupPrefix string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EnsureEKSNodeGroupMigrated", cluster, nodeGroupPrefix) + ret := m.ctrl.Call(m, "EnsureEKSNodegroupMigrated", cluster, nodeGroupPrefix) ret0, _ := ret[0].(error) return ret0 } -// EnsureEKSNodeGroupMigrated indicates an expected call of EnsureEKSNodeGroupMigrated -func (mr *MockAWSMockRecorder) EnsureEKSNodeGroupMigrated(cluster, nodeGroupPrefix interface{}) *gomock.Call { +// EnsureEKSNodegroupMigrated indicates an expected call of EnsureEKSNodegroupMigrated +func (mr *MockAWSMockRecorder) EnsureEKSNodegroupMigrated(cluster, nodeGroupPrefix interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureEKSNodeGroupMigrated", reflect.TypeOf((*MockAWS)(nil).EnsureEKSNodeGroupMigrated), cluster, nodeGroupPrefix) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureEKSNodegroupMigrated", reflect.TypeOf((*MockAWS)(nil).EnsureEKSNodegroupMigrated), cluster, nodeGroupPrefix) } // GetActiveEKSCluster mocks base method @@ -490,33 +490,33 @@ func (mr *MockAWSMockRecorder) GetActiveEKSCluster(clusterName interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveEKSCluster", reflect.TypeOf((*MockAWS)(nil).GetActiveEKSCluster), clusterName) } -// GetActiveEKSNodeGroup mocks base method -func (m *MockAWS) GetActiveEKSNodeGroup(clusterName, nodeGroupName string) (*types.Nodegroup, error) { +// GetActiveEKSNodegroup mocks base method +func (m *MockAWS) GetActiveEKSNodegroup(clusterName, nodeGroupName string) (*types.Nodegroup, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetActiveEKSNodeGroup", clusterName, nodeGroupName) + ret := m.ctrl.Call(m, "GetActiveEKSNodegroup", clusterName, nodeGroupName) ret0, _ := ret[0].(*types.Nodegroup) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetActiveEKSNodeGroup indicates an expected call of GetActiveEKSNodeGroup -func (mr *MockAWSMockRecorder) GetActiveEKSNodeGroup(clusterName, nodeGroupName interface{}) *gomock.Call { +// GetActiveEKSNodegroup indicates an expected call of GetActiveEKSNodegroup +func (mr *MockAWSMockRecorder) GetActiveEKSNodegroup(clusterName, nodeGroupName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveEKSNodeGroup", reflect.TypeOf((*MockAWS)(nil).GetActiveEKSNodeGroup), clusterName, nodeGroupName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveEKSNodegroup", reflect.TypeOf((*MockAWS)(nil).GetActiveEKSNodegroup), clusterName, nodeGroupName) } -// EnsureEKSNodeGroupDeleted mocks base method -func (m *MockAWS) EnsureEKSNodeGroupDeleted(clusterName, nodeGroupName string) error { +// EnsureEKSNodegroupDeleted mocks base method +func (m *MockAWS) EnsureEKSNodegroupDeleted(clusterName, nodeGroupName string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EnsureEKSNodeGroupDeleted", clusterName, nodeGroupName) + ret := m.ctrl.Call(m, "EnsureEKSNodegroupDeleted", clusterName, nodeGroupName) ret0, _ := ret[0].(error) return ret0 } -// EnsureEKSNodeGroupDeleted indicates an expected call of EnsureEKSNodeGroupDeleted -func (mr *MockAWSMockRecorder) EnsureEKSNodeGroupDeleted(clusterName, nodeGroupName interface{}) *gomock.Call { +// EnsureEKSNodegroupDeleted indicates an expected call of EnsureEKSNodegroupDeleted +func (mr *MockAWSMockRecorder) EnsureEKSNodegroupDeleted(clusterName, nodeGroupName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureEKSNodeGroupDeleted", reflect.TypeOf((*MockAWS)(nil).EnsureEKSNodeGroupDeleted), clusterName, nodeGroupName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureEKSNodegroupDeleted", reflect.TypeOf((*MockAWS)(nil).EnsureEKSNodegroupDeleted), clusterName, nodeGroupName) } // EnsureEKSClusterDeleted mocks base method @@ -562,33 +562,33 @@ func (mr *MockAWSMockRecorder) WaitForActiveEKSCluster(clusterName, timeout inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForActiveEKSCluster", reflect.TypeOf((*MockAWS)(nil).WaitForActiveEKSCluster), clusterName, timeout) } -// WaitForActiveEKSNodeGroup mocks base method -func (m *MockAWS) WaitForActiveEKSNodeGroup(clusterName, nodeGroupName string, timeout int) (*types.Nodegroup, error) { +// WaitForActiveEKSNodegroup mocks base method +func (m *MockAWS) WaitForActiveEKSNodegroup(clusterName, nodeGroupName string, timeout int) (*types.Nodegroup, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "WaitForActiveEKSNodeGroup", clusterName, nodeGroupName, timeout) + ret := m.ctrl.Call(m, "WaitForActiveEKSNodegroup", clusterName, nodeGroupName, timeout) ret0, _ := ret[0].(*types.Nodegroup) ret1, _ := ret[1].(error) return ret0, ret1 } -// WaitForActiveEKSNodeGroup indicates an expected call of WaitForActiveEKSNodeGroup -func (mr *MockAWSMockRecorder) WaitForActiveEKSNodeGroup(clusterName, nodeGroupName, timeout interface{}) *gomock.Call { +// WaitForActiveEKSNodegroup indicates an expected call of WaitForActiveEKSNodegroup +func (mr *MockAWSMockRecorder) WaitForActiveEKSNodegroup(clusterName, nodeGroupName, timeout interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForActiveEKSNodeGroup", reflect.TypeOf((*MockAWS)(nil).WaitForActiveEKSNodeGroup), clusterName, nodeGroupName, timeout) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForActiveEKSNodegroup", reflect.TypeOf((*MockAWS)(nil).WaitForActiveEKSNodegroup), clusterName, nodeGroupName, timeout) } -// WaitForEKSNodeGroupToBeDeleted mocks base method -func (m *MockAWS) WaitForEKSNodeGroupToBeDeleted(clusterName, nodeGroupName string, timeout int) error { +// WaitForEKSNodegroupToBeDeleted mocks base method +func (m *MockAWS) WaitForEKSNodegroupToBeDeleted(clusterName, nodeGroupName string, timeout int) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "WaitForEKSNodeGroupToBeDeleted", clusterName, nodeGroupName, timeout) + ret := m.ctrl.Call(m, "WaitForEKSNodegroupToBeDeleted", clusterName, nodeGroupName, timeout) ret0, _ := ret[0].(error) return ret0 } -// WaitForEKSNodeGroupToBeDeleted indicates an expected call of WaitForEKSNodeGroupToBeDeleted -func (mr *MockAWSMockRecorder) WaitForEKSNodeGroupToBeDeleted(clusterName, nodeGroupName, timeout interface{}) *gomock.Call { +// WaitForEKSNodegroupToBeDeleted indicates an expected call of WaitForEKSNodegroupToBeDeleted +func (mr *MockAWSMockRecorder) WaitForEKSNodegroupToBeDeleted(clusterName, nodeGroupName, timeout interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForEKSNodeGroupToBeDeleted", reflect.TypeOf((*MockAWS)(nil).WaitForEKSNodeGroupToBeDeleted), clusterName, nodeGroupName, timeout) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForEKSNodegroupToBeDeleted", reflect.TypeOf((*MockAWS)(nil).WaitForEKSNodegroupToBeDeleted), clusterName, nodeGroupName, timeout) } // WaitForEKSClusterToBeDeleted mocks base method diff --git a/internal/provisioner/eks_provisioner.go b/internal/provisioner/eks_provisioner.go index 8c841ab2c..25c48bedf 100644 --- a/internal/provisioner/eks_provisioner.go +++ b/internal/provisioner/eks_provisioner.go @@ -259,7 +259,7 @@ func (provisioner *EKSProvisioner) CreateNodegroups(cluster *model.Cluster) erro return } - _, err = provisioner.awsClient.EnsureEKSNodeGroup(cluster, ngPrefix) + _, err = provisioner.awsClient.EnsureEKSNodegroup(cluster, ngPrefix) if err != nil { logger.WithError(err).Errorf("Failed to create EKS nodegroup %s", ngMetadata.Name) errOccurred = true @@ -304,7 +304,7 @@ func (provisioner *EKSProvisioner) CheckNodegroupsCreated(cluster *model.Cluster wait := 300 logger.Infof("Waiting up to %d seconds for EKS nodegroup %s to become active...", wait, ngMetadata.Name) - nodeGroup, err := provisioner.awsClient.WaitForActiveEKSNodeGroup(eksMetadata.Name, ngMetadata.Name, wait) + nodeGroup, err := provisioner.awsClient.WaitForActiveEKSNodegroup(eksMetadata.Name, ngMetadata.Name, wait) if err != nil { logger.WithError(err).Errorf("Failed to wait for EKS nodegroup %s to become active", ngMetadata.Name) errOccurred = true @@ -349,7 +349,7 @@ func (provisioner *EKSProvisioner) CheckNodegroupsCreated(cluster *model.Cluster return true, nil } -// DeleteNodegroups deletes the EKS nodegroup. +// DeleteNodegroups deletes the EKS nodegroups. func (provisioner *EKSProvisioner) DeleteNodegroups(cluster *model.Cluster) error { logger := provisioner.logger.WithField("cluster", cluster.ID) @@ -373,7 +373,7 @@ func (provisioner *EKSProvisioner) DeleteNodegroups(cluster *model.Cluster) erro go func(ngPrefix string, ngMetadata model.NodeGroupMetadata) { defer wg.Done() - err := provisioner.awsClient.EnsureEKSNodeGroupDeleted(eksMetadata.Name, ngMetadata.Name) + err := provisioner.awsClient.EnsureEKSNodegroupDeleted(eksMetadata.Name, ngMetadata.Name) if err != nil { logger.WithError(err).Errorf("Failed to delete EKS nodedroup %s", ngMetadata.Name) errOccurred = true @@ -382,7 +382,7 @@ func (provisioner *EKSProvisioner) DeleteNodegroups(cluster *model.Cluster) erro wait := 600 logger.Infof("Waiting up to %d seconds for EKS nodegroup %s to be deleted...", wait, ngMetadata.Name) - err = provisioner.awsClient.WaitForEKSNodeGroupToBeDeleted(eksMetadata.Name, ngMetadata.Name, wait) + err = provisioner.awsClient.WaitForEKSNodegroupToBeDeleted(eksMetadata.Name, ngMetadata.Name, wait) if err != nil { logger.WithError(err).Errorf("Failed to delete EKS nodegroup %s", ngMetadata.Name) errOccurred = true @@ -476,7 +476,7 @@ func (provisioner *EKSProvisioner) UpgradeCluster(cluster *model.Cluster) error return } - err = provisioner.awsClient.EnsureEKSNodeGroupMigrated(cluster, ngPrefix) + err = provisioner.awsClient.EnsureEKSNodegroupMigrated(cluster, ngPrefix) if err != nil { logger.WithError(err).Errorf("Failed to migrate EKS nodegroup for %s", ngPrefix) errOccurred = true @@ -565,7 +565,7 @@ func (provisioner *EKSProvisioner) ResizeCluster(cluster *model.Cluster) error { ngMetadata.CopyMissingFieldsFrom(oldMetadata) changeRequest.NodeGroups[ngPrefix] = ngMetadata - oldEKSNodeGroup, err2 := provisioner.awsClient.GetActiveEKSNodeGroup(eksMetadata.Name, oldMetadata.Name) + oldEKSNodeGroup, err2 := provisioner.awsClient.GetActiveEKSNodegroup(eksMetadata.Name, oldMetadata.Name) if err2 != nil { logger.WithError(err2).Errorf("Failed to get the existing EKS nodegroup for %s", ngPrefix) errOccurred = true @@ -581,6 +581,7 @@ func (provisioner *EKSProvisioner) ResizeCluster(cluster *model.Cluster) error { oldMetadata.MinCount == ngMetadata.MinCount && oldMetadata.MaxCount == ngMetadata.MaxCount { logger.Debugf("No change in EKS nodegroup for %s", ngPrefix) + return } err = provisioner.prepareLaunchTemplate(cluster, ngPrefix, ngMetadata, logger) @@ -590,14 +591,14 @@ func (provisioner *EKSProvisioner) ResizeCluster(cluster *model.Cluster) error { return } - err = provisioner.awsClient.EnsureEKSNodeGroupMigrated(cluster, ngPrefix) + err = provisioner.awsClient.EnsureEKSNodegroupMigrated(cluster, ngPrefix) if err != nil { logger.WithError(err).Errorf("Failed to migrate EKS nodegroup for %s", ngPrefix) errOccurred = true return } - nodeGroup, err2 := provisioner.awsClient.GetActiveEKSNodeGroup(eksMetadata.Name, ngMetadata.Name) + nodeGroup, err2 := provisioner.awsClient.GetActiveEKSNodegroup(eksMetadata.Name, ngMetadata.Name) if err2 != nil { logger.WithError(err2).Errorf("Failed to get EKS nodegroup %s", ngMetadata.Name) errOccurred = true @@ -671,7 +672,7 @@ func (provisioner *EKSProvisioner) cleanupCluster(cluster *model.Cluster) error go func(ngPrefix string, ngMetadata model.NodeGroupMetadata) { defer wg.Done() - err = provisioner.awsClient.EnsureEKSNodeGroupDeleted(eksMetadata.Name, ngMetadata.Name) + err = provisioner.awsClient.EnsureEKSNodegroupDeleted(eksMetadata.Name, ngMetadata.Name) if err != nil { logger.WithError(err).Errorf("Failed to delete EKS nodegroup %s", ngMetadata.Name) errOccurred = true @@ -680,7 +681,7 @@ func (provisioner *EKSProvisioner) cleanupCluster(cluster *model.Cluster) error wait := 600 logger.Infof("Waiting up to %d seconds for nodegroup %s to be deleted...", wait, ngMetadata.Name) - err = provisioner.awsClient.WaitForEKSNodeGroupToBeDeleted(eksMetadata.Name, ngMetadata.Name, wait) + err = provisioner.awsClient.WaitForEKSNodegroupToBeDeleted(eksMetadata.Name, ngMetadata.Name, wait) if err != nil { logger.WithError(err).Errorf("Failed to delete EKS nodegroup %s", ngMetadata.Name) errOccurred = true @@ -695,7 +696,7 @@ func (provisioner *EKSProvisioner) cleanupCluster(cluster *model.Cluster) error return } - logger.Debugf("Successfully deleted EKS NodeGroup %s", ngMetadata.Name) + logger.Debugf("Successfully deleted EKS nodegroup %s", ngMetadata.Name) }(ng, meta) } @@ -794,7 +795,7 @@ func (provisioner *EKSProvisioner) getKubeConfigPath(cluster *model.Cluster) (st return "", errors.Wrap(err, "failed to get EKS cluster") } if eksCluster == nil { - return "", errors.Errorf("the EKS cluster %s does not exist", clusterName) + return "", errors.Errorf("No active EKS cluster found with name %s", clusterName) } kubeconfig, err := newEKSKubeConfig(eksCluster, provisioner.awsClient) diff --git a/internal/supervisor/cluster.go b/internal/supervisor/cluster.go index c42c79cfe..f7dd8f547 100644 --- a/internal/supervisor/cluster.go +++ b/internal/supervisor/cluster.go @@ -259,7 +259,7 @@ func (s *ClusterSupervisor) createNodegroups(cluster *model.Cluster, logger log. _, err = s.provisioner.GetClusterProvisioner(cluster.Provisioner).CheckNodegroupsCreated(cluster) if err != nil { - logger.WithError(err).Error("Failed to create nodegroups") + logger.WithError(err).Error("Failed to check if nodegroups are created") return model.ClusterStateNodegroupsCreationFailed } @@ -288,7 +288,7 @@ func (s *ClusterSupervisor) refreshClusterMetadata(cluster *model.Cluster, logge err = s.store.UpdateCluster(cluster) if err != nil { - logger.WithError(err).Error("Failed to update cluster metadata in store") + logger.WithError(err).Error("Failed to update cluster metadata") return model.ClusterStateRefreshMetadata } @@ -323,7 +323,7 @@ func (s *ClusterSupervisor) checkClusterCreated(cluster *model.Cluster, logger l return model.ClusterStateCreationFailed } if !ready { - logger.Debug("Cluster is not yet ready") + logger.Debug("Cluster not yet ready") return model.ClusterStateCreationInProgress } @@ -340,11 +340,11 @@ func (s *ClusterSupervisor) checkNodesCreated(cluster *model.Cluster, logger log ready, err := s.provisioner.GetClusterProvisioner(cluster.Provisioner).CheckNodegroupsCreated(cluster) if err != nil { - logger.WithError(err).Error("Failed to check if nodegroups creation finished") + logger.WithError(err).Error("Failed to check if nodegroups are created") return model.ClusterStateCreationFailed } if !ready { - logger.Debug("Cluster nodegroups are not ready yet") + logger.Debug("Cluster nodegroups not yet ready") return model.ClusterStateWaitingForNodes } diff --git a/internal/supervisor/installation_test.go b/internal/supervisor/installation_test.go index 3d6baac0c..8e50d8359 100644 --- a/internal/supervisor/installation_test.go +++ b/internal/supervisor/installation_test.go @@ -425,7 +425,7 @@ func (a *mockAWS) WaitForEKSClusterUpdateToBeCompleted(clusterName, updateID str return nil } -func (a *mockAWS) WaitForActiveEKSNodeGroup(clusterName, workerName string, timeout int) (*eksTypes.Nodegroup, error) { +func (a *mockAWS) WaitForActiveEKSNodegroup(clusterName, workerName string, timeout int) (*eksTypes.Nodegroup, error) { return nil, nil } @@ -433,7 +433,7 @@ func (a *mockAWS) WaitForEKSClusterToBeDeleted(clusterName string, timeout int) return nil } -func (a *mockAWS) WaitForEKSNodeGroupToBeDeleted(clusterName, workerName string, timeout int) error { +func (a *mockAWS) WaitForEKSNodegroupToBeDeleted(clusterName, workerName string, timeout int) error { return nil } @@ -441,15 +441,15 @@ func (a *mockAWS) EnsureEKSClusterUpdated(cluster *model.Cluster) (*eksTypes.Upd return nil, nil } -func (a *mockAWS) EnsureEKSNodeGroupMigrated(cluster *model.Cluster, ngPrefix string) error { +func (a *mockAWS) EnsureEKSNodegroupMigrated(cluster *model.Cluster, ngPrefix string) error { return nil } -func (a *mockAWS) GetActiveEKSNodeGroup(clusterName, workerName string) (*eksTypes.Nodegroup, error) { +func (a *mockAWS) GetActiveEKSNodegroup(clusterName, workerName string) (*eksTypes.Nodegroup, error) { return nil, nil } -func (a *mockAWS) EnsureEKSNodeGroupDeleted(clusterName, workerName string) error { +func (a *mockAWS) EnsureEKSNodegroupDeleted(clusterName, workerName string) error { return nil } @@ -506,7 +506,7 @@ func (a *mockAWS) EnsureEKSCluster(cluster *model.Cluster, resources aws.Cluster return &eksTypes.Cluster{}, nil } -func (a *mockAWS) EnsureEKSNodeGroup(cluster *model.Cluster, nodeGroupPrefix string) (*eksTypes.Nodegroup, error) { +func (a *mockAWS) EnsureEKSNodegroup(cluster *model.Cluster, nodeGroupPrefix string) (*eksTypes.Nodegroup, error) { return &eksTypes.Nodegroup{}, nil } diff --git a/internal/tools/aws/client.go b/internal/tools/aws/client.go index 0c5fa7f4e..059a56dc5 100644 --- a/internal/tools/aws/client.go +++ b/internal/tools/aws/client.go @@ -69,16 +69,16 @@ type AWS interface { EnsureEKSCluster(cluster *model.Cluster, resources ClusterResources) (*eksTypes.Cluster, error) EnsureEKSClusterUpdated(cluster *model.Cluster) (*eksTypes.Update, error) - EnsureEKSNodeGroup(cluster *model.Cluster, nodeGroupPrefix string) (*eksTypes.Nodegroup, error) - EnsureEKSNodeGroupMigrated(cluster *model.Cluster, nodeGroupPrefix string) error + EnsureEKSNodegroup(cluster *model.Cluster, nodeGroupPrefix string) (*eksTypes.Nodegroup, error) + EnsureEKSNodegroupMigrated(cluster *model.Cluster, nodeGroupPrefix string) error GetActiveEKSCluster(clusterName string) (*eksTypes.Cluster, error) - GetActiveEKSNodeGroup(clusterName, nodeGroupName string) (*eksTypes.Nodegroup, error) - EnsureEKSNodeGroupDeleted(clusterName, nodeGroupName string) error + GetActiveEKSNodegroup(clusterName, nodeGroupName string) (*eksTypes.Nodegroup, error) + EnsureEKSNodegroupDeleted(clusterName, nodeGroupName string) error EnsureEKSClusterDeleted(clusterName string) error InstallEKSAddons(cluster *model.Cluster) error WaitForActiveEKSCluster(clusterName string, timeout int) (*eksTypes.Cluster, error) - WaitForActiveEKSNodeGroup(clusterName, nodeGroupName string, timeout int) (*eksTypes.Nodegroup, error) - WaitForEKSNodeGroupToBeDeleted(clusterName, nodeGroupName string, timeout int) error + WaitForActiveEKSNodegroup(clusterName, nodeGroupName string, timeout int) (*eksTypes.Nodegroup, error) + WaitForEKSNodegroupToBeDeleted(clusterName, nodeGroupName string, timeout int) error WaitForEKSClusterToBeDeleted(clusterName string, timeout int) error WaitForEKSClusterUpdateToBeCompleted(clusterName, updateID string, timeout int) error diff --git a/internal/tools/aws/ec2.go b/internal/tools/aws/ec2.go index f9a5982fb..788566dcd 100644 --- a/internal/tools/aws/ec2.go +++ b/internal/tools/aws/ec2.go @@ -179,7 +179,7 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { eksCluster, err := a.getEKSCluster(data.ClusterName) if err != nil { - return errors.Wrap(err, "failed to get EKS Cluster") + return errors.Wrap(err, "failed to get EKS cluster") } userData := getLaunchTemplateUserData(eksCluster, data) @@ -205,14 +205,14 @@ func (a *Client) CreateLaunchTemplate(data *model.LaunchTemplateData) error { }) if err != nil { if IsErrorCode(err, "InvalidLaunchTemplateName.AlreadyExistsException") { - a.logger.Debugf("LaunchTemplate %s already exists", data.Name) + a.logger.Debugf("Launch template %s already exists", data.Name) return nil } - return errors.Wrap(err, "failed to create EKS LaunchTemplate") + return errors.Wrap(err, "failed to create EKS launch template") } if launchTemplate == nil || launchTemplate.LaunchTemplate == nil { - return errors.New("failed to create EKS LaunchTemplate") + return errors.New("failed to create EKS launch template") } return nil @@ -225,7 +225,7 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { eksCluster, err := a.getEKSCluster(data.ClusterName) if err != nil { - return errors.Wrap(err, "failed to get ESK Cluster") + return errors.Wrap(err, "failed to get ESK cluster") } userData := getLaunchTemplateUserData(eksCluster, data) @@ -253,11 +253,11 @@ func (a *Client) UpdateLaunchTemplate(data *model.LaunchTemplateData) error { if IsErrorCode(err, "InvalidLaunchTemplateName.NotFoundException") { return a.CreateLaunchTemplate(data) } - return errors.Wrap(err, "failed to create EKS LaunchTemplate version") + return errors.Wrap(err, "failed to create EKS launch template version") } if launchTemplate == nil || launchTemplate.LaunchTemplateVersion == nil { - return errors.New("failed to create ESK LaunchTemplate version") + return errors.New("failed to create ESK launch template version") } return nil @@ -270,7 +270,7 @@ func (a *Client) DeleteLaunchTemplate(launchTemplateName string) error { } if launchTemplate == nil { - a.logger.Debugf("LaunchTemplate %s not found, assuming deleted", launchTemplateName) + a.logger.Debugf("Launch template %s not found, assuming deleted", launchTemplateName) return nil } @@ -282,7 +282,7 @@ func (a *Client) DeleteLaunchTemplate(launchTemplateName string) error { a.logger.Debugf("launch template %s not found, assuming deleted", launchTemplateName) return nil } - return errors.Wrap(err, "failed to delete EKS LaunchTemplate") + return errors.Wrap(err, "failed to delete EKS launch template") } return nil diff --git a/internal/tools/aws/eks.go b/internal/tools/aws/eks.go index d826d8798..cd87bac76 100644 --- a/internal/tools/aws/eks.go +++ b/internal/tools/aws/eks.go @@ -70,7 +70,7 @@ func (c *Client) createEKSCluster(cluster *model.Cluster, resources ClusterResou out, err := c.Service().eks.CreateCluster(ctx, &input) if err != nil { - return nil, errors.Wrap(err, "failed to create EKS Cluster") + return nil, errors.Wrap(err, "failed to create EKS cluster") } return out.Cluster, nil @@ -83,7 +83,7 @@ func (a *Client) getEKSCluster(clusterName string) (*eksTypes.Cluster, error) { }) if err != nil { if !IsErrorResourceNotFound(err) { - return nil, errors.Wrap(err, "failed to describe EKS Cluster") + return nil, errors.Wrap(err, "failed to describe EKS cluster") } } @@ -162,11 +162,11 @@ func (c *Client) EnsureEKSClusterUpdated(cluster *model.Cluster) (*eksTypes.Upda } if eksCluster == nil { - return nil, errors.Errorf("requested EKS Cluster %s is not found", clusterName) + return nil, errors.Errorf("requested EKS cluster %s does not exist", clusterName) } if eksCluster.Status != eksTypes.ClusterStatusActive { - return nil, errors.Errorf("requested EKS Cluster %s is not active", clusterName) + return nil, errors.Errorf("requested EKS cluster %s is not active", clusterName) } eksMetadata := cluster.ProvisionerMetadataEKS @@ -180,7 +180,7 @@ func (c *Client) EnsureEKSClusterUpdated(cluster *model.Cluster) (*eksTypes.Upda }) if err != nil { - return nil, errors.Wrap(err, "failed to update EKS Cluster version") + return nil, errors.Wrap(err, "failed to update EKS cluster version") } return output.Update, nil @@ -191,7 +191,7 @@ func (a *Client) createEKSNodeGroup(cluster *model.Cluster, ngPrefix string) (*e eksMetadata := cluster.ProvisionerMetadataEKS changeRequest := eksMetadata.ChangeRequest if changeRequest == nil { - return nil, errors.New("metadata ChangeRequest is not set") + return nil, errors.New("metadata ChangeRequest is nil") } clusterResource, err := a.GetVpcResourcesByVpcID(changeRequest.VPC, a.logger) @@ -272,7 +272,7 @@ func (a *Client) createEKSNodeGroup(cluster *model.Cluster, ngPrefix string) (*e out, err := a.Service().eks.CreateNodegroup(context.TODO(), &nodeGroupReq) if err != nil { - return nil, errors.Wrap(err, "failed to create EKS Nodegroup") + return nil, errors.Wrap(err, "failed to create EKS nodegroup") } return out.Nodegroup, nil @@ -286,7 +286,7 @@ func (c *Client) getEKSNodeGroup(clusterName, nodegroupName string) (*eksTypes.N }) if err != nil { if !IsErrorResourceNotFound(err) { - return nil, errors.Wrap(err, "failed to describe EKS Nodegroup") + return nil, errors.Wrap(err, "failed to describe EKS nodegroup") } } @@ -297,8 +297,8 @@ func (c *Client) getEKSNodeGroup(clusterName, nodegroupName string) (*eksTypes.N return nil, nil } -// EnsureEKSNodeGroup ensures EKS cluster node groups are created. -func (c *Client) EnsureEKSNodeGroup(cluster *model.Cluster, ngPrefix string) (*eksTypes.Nodegroup, error) { +// EnsureEKSNodegroup ensures EKS nodegroup is created. +func (c *Client) EnsureEKSNodegroup(cluster *model.Cluster, ngPrefix string) (*eksTypes.Nodegroup, error) { clusterName := cluster.ProvisionerMetadataEKS.Name @@ -312,7 +312,7 @@ func (c *Client) EnsureEKSNodeGroup(cluster *model.Cluster, ngPrefix string) (*e nodeGroup, err := c.getEKSNodeGroup(clusterName, ngChangeRequest.Name) if err != nil { - return nil, errors.Wrapf(err, "failed to get an EKS Nodegroup %s", ngChangeRequest.Name) + return nil, errors.Wrapf(err, "failed to get an EKS nodegroup %s", ngChangeRequest.Name) } if nodeGroup != nil { @@ -322,8 +322,8 @@ func (c *Client) EnsureEKSNodeGroup(cluster *model.Cluster, ngPrefix string) (*e return c.createEKSNodeGroup(cluster, ngPrefix) } -// EnsureEKSNodeGroupMigrated updates EKS cluster node group. -func (c *Client) EnsureEKSNodeGroupMigrated(cluster *model.Cluster, ngPrefix string) error { +// EnsureEKSNodegroupMigrated updates EKS nodegroup. +func (c *Client) EnsureEKSNodegroupMigrated(cluster *model.Cluster, ngPrefix string) error { logger := c.logger.WithField("cluster", cluster.ID) eksMetadata := cluster.ProvisionerMetadataEKS @@ -344,26 +344,26 @@ func (c *Client) EnsureEKSNodeGroupMigrated(cluster *model.Cluster, ngPrefix str _, err := c.createEKSNodeGroup(cluster, ngPrefix) if err != nil { - return errors.Wrapf(err, "failed to create a new EKS Nodegroup %s", ngChangeRequest.Name) + return errors.Wrapf(err, "failed to create a new EKS nodegroup %s", ngChangeRequest.Name) } wait := 600 // seconds - logger.Infof("Waiting up to %d seconds for EKS Nodegroup %s to become active...", wait, ngChangeRequest.Name) + logger.Infof("Waiting up to %d seconds for EKS nodegroup %s to become active...", wait, ngChangeRequest.Name) - _, err = c.WaitForActiveEKSNodeGroup(eksMetadata.Name, ngChangeRequest.Name, wait) + _, err = c.WaitForActiveEKSNodegroup(eksMetadata.Name, ngChangeRequest.Name, wait) if err != nil { return err } - logger.Debugf("Deleting the old EKS Nodegroup %s", oldNodeGroupName) + logger.Debugf("Deleting the old EKS nodegroup %s", oldNodeGroupName) - err = c.EnsureEKSNodeGroupDeleted(clusterName, oldNodeGroupName) + err = c.EnsureEKSNodegroupDeleted(clusterName, oldNodeGroupName) if err != nil { - return errors.Wrapf(err, "failed to delete the old EKS Nodegroup %s", oldNodeGroupName) + return errors.Wrapf(err, "failed to delete the old EKS nodegroup %s", oldNodeGroupName) } - logger.Infof("Waiting up to %d seconds for EKS Nodegroup %s to be deleted...", wait, oldNodeGroupName) - err = c.WaitForEKSNodeGroupToBeDeleted(eksMetadata.Name, oldNodeGroupName, wait) + logger.Infof("Waiting up to %d seconds for EKS nodegroup %s to be deleted...", wait, oldNodeGroupName) + err = c.WaitForEKSNodegroupToBeDeleted(eksMetadata.Name, oldNodeGroupName, wait) if err != nil { return err } @@ -377,7 +377,7 @@ func (a *Client) EnsureEKSClusterDeleted(clusterName string) error { eksCluster, err := a.getEKSCluster(clusterName) if err != nil { - return errors.Wrap(err, "failed to describe EKS Cluster") + return errors.Wrap(err, "failed to describe EKS cluster") } if eksCluster == nil { @@ -390,48 +390,48 @@ func (a *Client) EnsureEKSClusterDeleted(clusterName string) error { } if eksCluster.Status == eksTypes.ClusterStatusFailed { - return errors.New("requested EKS Cluster is in failed state") + return errors.New("requested EKS cluster is in failed state") } delInput := &eks.DeleteClusterInput{Name: aws.String(clusterName)} _, err = a.Service().eks.DeleteCluster(ctx, delInput) if err != nil { - return errors.Wrap(err, "failed to trigger EKS Cluster deletion") + return errors.Wrap(err, "failed to trigger EKS cluster deletion") } // Cluster just started deletion return nil } -// EnsureEKSNodeGroupDeleted ensures EKS node groups are deleted. -func (a *Client) EnsureEKSNodeGroupDeleted(clusterName, nodegroupName string) error { +// EnsureEKSNodegroupDeleted ensures EKS nodegroup is deleted. +func (a *Client) EnsureEKSNodegroupDeleted(clusterName, nodegroupName string) error { if nodegroupName == "" { return nil } - nodeGroups, err := a.getEKSNodeGroup(clusterName, nodegroupName) + nodeGroup, err := a.getEKSNodeGroup(clusterName, nodegroupName) if err != nil { - return errors.Wrap(err, "failed to get EKS Nodegroup") + return errors.Wrap(err, "failed to get EKS nodegroup") } - // Node groups deleted, we can return - if nodeGroups == nil { + + if nodeGroup == nil { return nil } - if nodeGroups.Status == eksTypes.NodegroupStatusDeleting { + if nodeGroup.Status == eksTypes.NodegroupStatusDeleting { return nil } - if nodeGroups.Status == eksTypes.NodegroupStatusDeleteFailed { - return errors.Wrapf(err, "failed to delete EKS Nodegroup %s", nodegroupName) + if nodeGroup.Status == eksTypes.NodegroupStatusDeleteFailed { + return errors.Wrapf(err, "failed to delete EKS nodegroup %s", nodegroupName) } _, err = a.Service().eks.DeleteNodegroup(context.TODO(), &eks.DeleteNodegroupInput{ ClusterName: aws.String(clusterName), - NodegroupName: nodeGroups.NodegroupName, + NodegroupName: nodeGroup.NodegroupName, }) if err != nil { - return errors.Wrap(err, "failed to delete EKS Nodegroup") + return errors.Wrap(err, "failed to delete EKS nodegroup") } return nil @@ -441,7 +441,7 @@ func (a *Client) EnsureEKSNodeGroupDeleted(clusterName, nodegroupName string) er func (c *Client) GetActiveEKSCluster(clusterName string) (*eksTypes.Cluster, error) { cluster, err := c.getEKSCluster(clusterName) if err != nil { - return nil, errors.Wrap(err, "failed to get EKS Cluster") + return nil, errors.Wrap(err, "failed to get EKS cluster") } if cluster == nil { @@ -449,7 +449,7 @@ func (c *Client) GetActiveEKSCluster(clusterName string) (*eksTypes.Cluster, err } if cluster.Status == eksTypes.ClusterStatusFailed { - return nil, errors.New("requested EKS Cluster is in failed state") + return nil, errors.New("requested EKS cluster is in failed state") } if cluster.Status == eksTypes.ClusterStatusActive { @@ -459,11 +459,11 @@ func (c *Client) GetActiveEKSCluster(clusterName string) (*eksTypes.Cluster, err return nil, nil } -// GetActiveEKSNodeGroup returns the EKS node group if active. -func (c *Client) GetActiveEKSNodeGroup(clusterName, workerName string) (*eksTypes.Nodegroup, error) { +// GetActiveEKSNodegroup returns the EKS nodegroup if active. +func (c *Client) GetActiveEKSNodegroup(clusterName, workerName string) (*eksTypes.Nodegroup, error) { nodeGroup, err := c.getEKSNodeGroup(clusterName, workerName) if err != nil { - return nil, errors.Wrap(err, "failed to get EKS Nodegroup") + return nil, errors.Wrap(err, "failed to get EKS nodegroup") } if nodeGroup == nil { @@ -471,7 +471,7 @@ func (c *Client) GetActiveEKSNodeGroup(clusterName, workerName string) (*eksType } if nodeGroup.Status == eksTypes.NodegroupStatusCreateFailed { - return nil, errors.New("failed to create EKS Nodegroup") + return nil, errors.New("failed to create EKS nodegroup") } if nodeGroup.Status == eksTypes.NodegroupStatusActive { @@ -491,11 +491,11 @@ func (c *Client) WaitForActiveEKSCluster(clusterName string, timeout int) (*eksT for { select { case <-timeoutTimer.C: - return nil, errors.New("timed out waiting for EKS Cluster to become active") + return nil, errors.New("timed out waiting for EKS cluster to become active") case <-tick.C: eksCluster, err := c.GetActiveEKSCluster(clusterName) if err != nil { - return nil, errors.Wrap(err, "failed to check if EKS Cluster is active") + return nil, errors.Wrap(err, "failed to check if EKS cluster is active") } if eksCluster != nil { return eksCluster, nil @@ -504,7 +504,7 @@ func (c *Client) WaitForActiveEKSCluster(clusterName string, timeout int) (*eksT } } -func (c *Client) WaitForActiveEKSNodeGroup(clusterName, nodeGroupName string, timeout int) (*eksTypes.Nodegroup, error) { +func (c *Client) WaitForActiveEKSNodegroup(clusterName, nodeGroupName string, timeout int) (*eksTypes.Nodegroup, error) { timeoutTimer := time.NewTimer(time.Duration(timeout) * time.Second) defer timeoutTimer.Stop() tick := time.NewTicker(5 * time.Second) @@ -513,11 +513,11 @@ func (c *Client) WaitForActiveEKSNodeGroup(clusterName, nodeGroupName string, ti for { select { case <-timeoutTimer.C: - return nil, errors.New("timed out waiting for EKS Nodegroup to become active") + return nil, errors.New("timed out waiting for EKS nodegroup to become active") case <-tick.C: - nodeGroup, err := c.GetActiveEKSNodeGroup(clusterName, nodeGroupName) + nodeGroup, err := c.GetActiveEKSNodegroup(clusterName, nodeGroupName) if err != nil { - return nil, errors.Wrap(err, "failed to check if EKS Nodegroup is active") + return nil, errors.Wrap(err, "failed to check if EKS nodegroup is active") } if nodeGroup != nil { return nodeGroup, nil @@ -526,7 +526,7 @@ func (c *Client) WaitForActiveEKSNodeGroup(clusterName, nodeGroupName string, ti } } -func (c *Client) WaitForEKSNodeGroupToBeDeleted(clusterName, workerName string, timeout int) error { +func (c *Client) WaitForEKSNodegroupToBeDeleted(clusterName, workerName string, timeout int) error { timeoutTimer := time.NewTimer(time.Duration(timeout) * time.Second) defer timeoutTimer.Stop() tick := time.NewTicker(5 * time.Second) @@ -535,11 +535,11 @@ func (c *Client) WaitForEKSNodeGroupToBeDeleted(clusterName, workerName string, for { select { case <-timeoutTimer.C: - return errors.New("timed out waiting for EKS Nodegroup to be deleted") + return errors.New("timed out waiting for EKS nodegroup to be deleted") case <-tick.C: nodeGroup, err := c.getEKSNodeGroup(clusterName, workerName) if err != nil { - return errors.Wrap(err, "failed to describe EKS Nodegroup") + return errors.Wrap(err, "failed to describe EKS nodegroup") } if nodeGroup == nil { return nil @@ -557,11 +557,11 @@ func (c *Client) WaitForEKSClusterToBeDeleted(clusterName string, timeout int) e for { select { case <-timeoutTimer.C: - return errors.New("timed out waiting for EKS Cluster to become ready") + return errors.New("timed out waiting for EKS cluster to become ready") case <-tick.C: eksCluster, err := c.getEKSCluster(clusterName) if err != nil { - return errors.Wrap(err, "failed to describe EKS Cluster") + return errors.Wrap(err, "failed to describe EKS cluster") } if eksCluster == nil { return nil @@ -579,15 +579,15 @@ func (c *Client) WaitForEKSClusterUpdateToBeCompleted(clusterName, updateID stri for { select { case <-timeoutTimer.C: - return errors.New("timed out waiting for EKS Cluster update to be completed") + return errors.New("timed out waiting for EKS cluster update to be completed") case <-tick.C: updateStatus, err := c.getEKSClusterUpdateStatus(clusterName, updateID) if err != nil { - return errors.Wrap(err, "failed to describe EKS Cluster") + return errors.Wrap(err, "failed to describe EKS cluster") } if updateStatus == eksTypes.UpdateStatusFailed { - return errors.New("failed to update EKS Cluster") + return errors.New("failed to update EKS cluster") } if updateStatus == eksTypes.UpdateStatusSuccessful { @@ -603,7 +603,7 @@ func (c *Client) getEKSClusterUpdateStatus(clusterName, updateID string) (eksTyp UpdateId: ptr.String(updateID), }) if err != nil { - return "", errors.Wrap(err, "failed to describe update for EKS Cluster") + return "", errors.Wrap(err, "failed to describe update for EKS cluster") } return output.Update.Status, nil