From 81f5ca31c231a11174aec281a9cda93f08c69ebd Mon Sep 17 00:00:00 2001 From: Richard Case Date: Wed, 14 Dec 2022 13:00:22 +0000 Subject: [PATCH] feat: added GCPManagedCluster controller implementation Signed-off-by: Richard Case --- cloud/scope/cluster.go | 38 +-- cloud/scope/credentials.go | 63 ++++ cloud/scope/managedcluster.go | 287 ++++++++++++++++++ ...e.cluster.x-k8s.io_gcpmanagedclusters.yaml | 15 + config/manager/manager.yaml | 1 + exp/api/v1beta1/gcpmanagedcluster_types.go | 8 + exp/api/v1beta1/zz_generated.deepcopy.go | 5 + .../gcpmanagedcluster_controller.go | 170 ++++++++++- feature/feature.go | 17 +- main.go | 105 +++++-- 10 files changed, 630 insertions(+), 79 deletions(-) create mode 100644 cloud/scope/credentials.go create mode 100644 cloud/scope/managedcluster.go diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index 265562ff7a..02d31a0b2a 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -24,9 +24,6 @@ import ( "github.com/pkg/errors" "google.golang.org/api/compute/v1" - "google.golang.org/api/option" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" infrav1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-gcp/cloud" @@ -54,7 +51,7 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc } if params.GCPServices.Compute == nil { - computeSvc, err := createComputeService(ctx, params) + computeSvc, err := createComputeService(ctx, params.GCPCluster.Spec.CredentialsRef, params.Client) if err != nil { return nil, errors.Errorf("failed to create gcp compute client: %v", err) } @@ -76,39 +73,6 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc }, nil } -func createComputeService(ctx context.Context, params ClusterScopeParams) (*compute.Service, error) { - if params.GCPCluster.Spec.CredentialsRef == nil { - computeSvc, err := compute.NewService(ctx) - if err != nil { - return nil, errors.Errorf("failed to create gcp compute client: %v", err) - } - - return computeSvc, nil - } - - secretRefName := types.NamespacedName{ - Name: params.GCPCluster.Spec.CredentialsRef.Name, - Namespace: params.GCPCluster.Spec.CredentialsRef.Namespace, - } - - credSecret := &corev1.Secret{} - if err := params.Client.Get(ctx, secretRefName, credSecret); err != nil { - return nil, fmt.Errorf("getting credentials secret %s\\%s: %w", secretRefName.Namespace, secretRefName.Name, err) - } - - rawData, ok := credSecret.Data["credentials"] - if !ok { - return nil, errors.New("no credentials key in secret") - } - - computeSvc, err := compute.NewService(ctx, option.WithCredentialsJSON(rawData)) - if err != nil { - return nil, errors.Errorf("failed to create gcp compute client with credentials secret: %v", err) - } - - return computeSvc, nil -} - // ClusterScope defines the basic context for an actuator to operate upon. type ClusterScope struct { client client.Client diff --git a/cloud/scope/credentials.go b/cloud/scope/credentials.go new file mode 100644 index 0000000000..28eaa70b83 --- /dev/null +++ b/cloud/scope/credentials.go @@ -0,0 +1,63 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scope + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + "google.golang.org/api/compute/v1" + "google.golang.org/api/option" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + infrav1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func createComputeService(ctx context.Context, credentialsRef *infrav1.ObjectReference, crClient client.Client) (*compute.Service, error) { + if credentialsRef == nil { + computeSvc, err := compute.NewService(ctx) + if err != nil { + return nil, errors.Errorf("failed to create gcp compute client: %v", err) + } + + return computeSvc, nil + } + + secretRefName := types.NamespacedName{ + Name: credentialsRef.Name, + Namespace: credentialsRef.Namespace, + } + + credSecret := &corev1.Secret{} + if err := crClient.Get(ctx, secretRefName, credSecret); err != nil { + return nil, fmt.Errorf("getting credentials secret %s\\%s: %w", secretRefName.Namespace, secretRefName.Name, err) + } + + rawData, ok := credSecret.Data["credentials"] + if !ok { + return nil, errors.New("no credentials key in secret") + } + + computeSvc, err := compute.NewService(ctx, option.WithCredentialsJSON(rawData)) + if err != nil { + return nil, errors.Errorf("failed to create gcp compute client with credentials secret: %v", err) + } + + return computeSvc, nil +} diff --git a/cloud/scope/managedcluster.go b/cloud/scope/managedcluster.go new file mode 100644 index 0000000000..5b694f5aaf --- /dev/null +++ b/cloud/scope/managedcluster.go @@ -0,0 +1,287 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scope + +import ( + "context" + "fmt" + "strconv" + + "github.com/pkg/errors" + "google.golang.org/api/compute/v1" + "k8s.io/utils/pointer" + infrav1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-gcp/cloud" + infrav1exp "sigs.k8s.io/cluster-api-provider-gcp/exp/api/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ManagedClusterScopeParams defines the input parameters used to create a new Scope. +type ManagedClusterScopeParams struct { + GCPServices + Client client.Client + Cluster *clusterv1.Cluster + GCPManagedCluster *infrav1exp.GCPManagedCluster + GCPManagedControlPlane *infrav1exp.GCPManagedControlPlane +} + +// NewManagedClusterScope creates a new Scope from the supplied parameters. +// This is meant to be called for each reconcile iteration. +func NewManagedClusterScope(ctx context.Context, params ManagedClusterScopeParams) (*ManagedClusterScope, error) { + if params.Cluster == nil { + return nil, errors.New("failed to generate new scope from nil Cluster") + } + if params.GCPManagedCluster == nil { + return nil, errors.New("failed to generate new scope from nil GCPManagedCluster") + } + if params.GCPManagedControlPlane == nil { + return nil, errors.New("failed to generate new scope from nil GCPManagedControlPlane") + } + + if params.GCPServices.Compute == nil { + computeSvc, err := createComputeService(ctx, params.GCPManagedCluster.Spec.CredentialsRef, params.Client) + if err != nil { + return nil, errors.Errorf("failed to create gcp compute client: %v", err) + } + + params.GCPServices.Compute = computeSvc + } + + helper, err := patch.NewHelper(params.GCPManagedCluster, params.Client) + if err != nil { + return nil, errors.Wrap(err, "failed to init patch helper") + } + + return &ManagedClusterScope{ + client: params.Client, + Cluster: params.Cluster, + GCPManagedCluster: params.GCPManagedCluster, + GCPManagedControlPlane: params.GCPManagedControlPlane, + GCPServices: params.GCPServices, + patchHelper: helper, + }, nil +} + +// ManagedClusterScope defines the basic context for an actuator to operate upon. +type ManagedClusterScope struct { + client client.Client + patchHelper *patch.Helper + + Cluster *clusterv1.Cluster + GCPManagedCluster *infrav1exp.GCPManagedCluster + GCPManagedControlPlane *infrav1exp.GCPManagedControlPlane + GCPServices +} + +// ANCHOR: ClusterGetter + +// Cloud returns initialized cloud. +func (s *ManagedClusterScope) Cloud() cloud.Cloud { + return newCloud(s.Project(), s.GCPServices) +} + +// Project returns the current project name. +func (s *ManagedClusterScope) Project() string { + return s.GCPManagedCluster.Spec.Project +} + +// Region returns the cluster region. +func (s *ManagedClusterScope) Region() string { + return s.GCPManagedCluster.Spec.Region +} + +// Name returns the cluster name. +func (s *ManagedClusterScope) Name() string { + return s.Cluster.Name +} + +// Namespace returns the cluster namespace. +func (s *ManagedClusterScope) Namespace() string { + return s.Cluster.Namespace +} + +// NetworkName returns the cluster network unique identifier. +func (s *ManagedClusterScope) NetworkName() string { + return pointer.StringDeref(s.GCPManagedCluster.Spec.Network.Name, "default") +} + +// NetworkLink returns the partial URL for the network. +func (s *ManagedClusterScope) NetworkLink() string { + return fmt.Sprintf("projects/%s/global/networks/%s", s.Project(), s.NetworkName()) +} + +// Network returns the cluster network object. +func (s *ManagedClusterScope) Network() *infrav1.Network { + return &s.GCPManagedCluster.Status.Network +} + +// AdditionalLabels returns the cluster additional labels. +func (s *ManagedClusterScope) AdditionalLabels() infrav1.Labels { + return s.GCPManagedCluster.Spec.AdditionalLabels +} + +// ControlPlaneEndpoint returns the cluster control-plane endpoint. +func (s *ManagedClusterScope) ControlPlaneEndpoint() clusterv1.APIEndpoint { + endpoint := s.GCPManagedCluster.Spec.ControlPlaneEndpoint + endpoint.Port = pointer.Int32Deref(s.Cluster.Spec.ClusterNetwork.APIServerPort, 443) + return endpoint +} + +// FailureDomains returns the cluster failure domains. +func (s *ManagedClusterScope) FailureDomains() clusterv1.FailureDomains { + return s.GCPManagedCluster.Status.FailureDomains +} + +// ANCHOR_END: ClusterGetter + +// ANCHOR: ClusterSetter + +// SetReady sets cluster ready status. +func (s *ManagedClusterScope) SetReady() { + s.GCPManagedCluster.Status.Ready = true +} + +// SetFailureDomains sets cluster failure domains. +func (s *ManagedClusterScope) SetFailureDomains(fd clusterv1.FailureDomains) { + s.GCPManagedCluster.Status.FailureDomains = fd +} + +// SetControlPlaneEndpoint sets cluster control-plane endpoint. +func (s *ManagedClusterScope) SetControlPlaneEndpoint(endpoint clusterv1.APIEndpoint) { + s.GCPManagedCluster.Spec.ControlPlaneEndpoint = endpoint +} + +// ANCHOR_END: ClusterSetter + +// ANCHOR: ClusterNetworkSpec + +// NetworkSpec returns google compute network spec. +func (s *ManagedClusterScope) NetworkSpec() *compute.Network { + createSubnet := pointer.BoolDeref(s.GCPManagedCluster.Spec.Network.AutoCreateSubnetworks, true) + network := &compute.Network{ + Name: s.NetworkName(), + Description: infrav1.ClusterTagKey(s.Name()), + AutoCreateSubnetworks: createSubnet, + ForceSendFields: []string{"AutoCreateSubnetworks"}, + } + + return network +} + +// NatRouterSpec returns google compute nat router spec. +func (s *ManagedClusterScope) NatRouterSpec() *compute.Router { + networkSpec := s.NetworkSpec() + return &compute.Router{ + Name: fmt.Sprintf("%s-%s", networkSpec.Name, "router"), + Nats: []*compute.RouterNat{ + { + Name: fmt.Sprintf("%s-%s", networkSpec.Name, "nat"), + NatIpAllocateOption: "AUTO_ONLY", + SourceSubnetworkIpRangesToNat: "ALL_SUBNETWORKS_ALL_IP_RANGES", + }, + }, + } +} + +// ANCHOR_END: ClusterNetworkSpec + +// SubnetSpecs returns google compute subnets spec. +func (s *ManagedClusterScope) SubnetSpecs() []*compute.Subnetwork { + subnets := []*compute.Subnetwork{} + for _, subnetwork := range s.GCPManagedCluster.Spec.Network.Subnets { + secondaryIPRanges := []*compute.SubnetworkSecondaryRange{} + for _, secondaryCidrBlock := range subnetwork.SecondaryCidrBlocks { + secondaryIPRanges = append(secondaryIPRanges, &compute.SubnetworkSecondaryRange{IpCidrRange: secondaryCidrBlock}) + } + subnets = append(subnets, &compute.Subnetwork{ + Name: subnetwork.Name, + Region: subnetwork.Region, + EnableFlowLogs: pointer.BoolDeref(subnetwork.EnableFlowLogs, false), + PrivateIpGoogleAccess: pointer.BoolDeref(subnetwork.PrivateGoogleAccess, false), + IpCidrRange: subnetwork.CidrBlock, + SecondaryIpRanges: secondaryIPRanges, + Description: pointer.StringDeref(subnetwork.Description, infrav1.ClusterTagKey(s.Name())), + Network: s.NetworkLink(), + Purpose: pointer.StringDeref(subnetwork.Purpose, "PRIVATE_RFC_1918"), + Role: "ACTIVE", + }) + } + + return subnets +} + +// ANCHOR: ClusterFirewallSpec + +// FirewallRulesSpec returns google compute firewall spec. +func (s *ManagedClusterScope) FirewallRulesSpec() []*compute.Firewall { + firewallRules := []*compute.Firewall{ + { + Name: fmt.Sprintf("allow-%s-healthchecks", s.Name()), + Network: s.NetworkLink(), + Allowed: []*compute.FirewallAllowed{ + { + IPProtocol: "TCP", + Ports: []string{ + strconv.FormatInt(6443, 10), + }, + }, + }, + Direction: "INGRESS", + SourceRanges: []string{ + "35.191.0.0/16", + "130.211.0.0/22", + }, + TargetTags: []string{ + fmt.Sprintf("%s-control-plane", s.Name()), + }, + }, + { + Name: fmt.Sprintf("allow-%s-cluster", s.Name()), + Network: s.NetworkLink(), + Allowed: []*compute.FirewallAllowed{ + { + IPProtocol: "all", + }, + }, + Direction: "INGRESS", + SourceTags: []string{ + fmt.Sprintf("%s-control-plane", s.Name()), + fmt.Sprintf("%s-node", s.Name()), + }, + TargetTags: []string{ + fmt.Sprintf("%s-control-plane", s.Name()), + fmt.Sprintf("%s-node", s.Name()), + }, + }, + } + + return firewallRules +} + +// ANCHOR_END: ClusterFirewallSpec + +// PatchObject persists the cluster configuration and status. +func (s *ManagedClusterScope) PatchObject() error { + return s.patchHelper.Patch(context.TODO(), s.GCPManagedCluster) +} + +// Close closes the current scope persisting the cluster configuration and status. +func (s *ManagedClusterScope) Close() error { + return s.PatchObject() +} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml index 4270430654..b775cc6cbe 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml @@ -79,6 +79,21 @@ spec: - host - port type: object + credentialsRef: + description: CredentialsRef is a reference to a Secret that contains + the credentials to use for provisioning this cluster. If not supplied + then the credentials of the controller will be used. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + required: + - name + - namespace + type: object network: description: NetworkSpec encapsulates all things related to the GCP network. diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d75d04a07b..28f6809272 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -21,6 +21,7 @@ spec: containers: - args: - --leader-elect + - --feature-gates=GKE=${EXP_CAPG_GKE:=false} - "--metrics-bind-addr=localhost:8080" image: controller:latest imagePullPolicy: IfNotPresent diff --git a/exp/api/v1beta1/gcpmanagedcluster_types.go b/exp/api/v1beta1/gcpmanagedcluster_types.go index 6564b701da..a78a146302 100644 --- a/exp/api/v1beta1/gcpmanagedcluster_types.go +++ b/exp/api/v1beta1/gcpmanagedcluster_types.go @@ -32,18 +32,26 @@ const ( type GCPManagedClusterSpec struct { // Project is the name of the project to deploy the cluster to. Project string `json:"project"` + // The GCP Region the cluster lives in. Region string `json:"region"` + // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. // +optional ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` // NetworkSpec encapsulates all things related to the GCP network. // +optional Network infrav1.NetworkSpec `json:"network"` + // AdditionalLabels is an optional set of tags to add to GCP resources managed by the GCP provider, in addition to the // ones added by default. // +optional AdditionalLabels infrav1.Labels `json:"additionalLabels,omitempty"` + + // CredentialsRef is a reference to a Secret that contains the credentials to use for provisioning this cluster. If not + // supplied then the credentials of the controller will be used. + // +optional + CredentialsRef *infrav1.ObjectReference `json:"credentialsRef,omitempty"` } // GCPManagedClusterStatus defines the observed state of GCPManagedCluster. diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index cd997f930f..b430947c63 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -98,6 +98,11 @@ func (in *GCPManagedClusterSpec) DeepCopyInto(out *GCPManagedClusterSpec) { (*out)[key] = val } } + if in.CredentialsRef != nil { + in, out := &in.CredentialsRef, &out.CredentialsRef + *out = new(apiv1beta1.ObjectReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCPManagedClusterSpec. diff --git a/exp/controllers/gcpmanagedcluster_controller.go b/exp/controllers/gcpmanagedcluster_controller.go index ae0e7624a7..236042b584 100644 --- a/exp/controllers/gcpmanagedcluster_controller.go +++ b/exp/controllers/gcpmanagedcluster_controller.go @@ -19,19 +19,30 @@ package controllers import ( "context" "fmt" + "time" + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/filter" + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" "k8s.io/klog/v2" + "sigs.k8s.io/cluster-api-provider-gcp/cloud" + "sigs.k8s.io/cluster-api-provider-gcp/cloud/scope" + "sigs.k8s.io/cluster-api-provider-gcp/cloud/services/compute/networks" + "sigs.k8s.io/cluster-api-provider-gcp/cloud/services/compute/subnets" infrav1exp "sigs.k8s.io/cluster-api-provider-gcp/exp/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-gcp/util/reconciler" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/cluster-api/util/predicates" + "sigs.k8s.io/cluster-api/util/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/source" @@ -40,24 +51,90 @@ import ( // GCPManagedClusterReconciler reconciles a GCPManagedCluster object. type GCPManagedClusterReconciler struct { client.Client - Recorder record.EventRecorder WatchFilterValue string Scheme *runtime.Scheme + ReconcileTimeout time.Duration } //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=gcpmanagedclusters,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=gcpmanagedclusters/status,verbs=get;update;patch //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=gcpmanagedclusters/finalizers,verbs=update +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=gcpmanagedcontrolplanes,verbs=get;list;watch //+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status,verbs=get;list;watch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. -func (r *GCPManagedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) +func (r *GCPManagedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { + ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultedLoopTimeout(r.ReconcileTimeout)) + defer cancel() - // TODO(user): your logic here + log := log.FromContext(ctx) - return ctrl.Result{}, nil + gcpCluster := &infrav1exp.GCPManagedCluster{} + err := r.Get(ctx, req.NamespacedName, gcpCluster) + if err != nil { + if apierrors.IsNotFound(err) { + log.Info("GCPManagedCluster resource not found or already deleted") + return ctrl.Result{}, nil + } + + log.Error(err, "Unable to fetch GCPManagedCluster resource") + return ctrl.Result{}, err + } + + // Fetch the Cluster. + cluster, err := util.GetOwnerCluster(ctx, r.Client, gcpCluster.ObjectMeta) + if err != nil { + log.Error(err, "Failed to get owner cluster") + return ctrl.Result{}, err + } + if cluster == nil { + log.Info("Cluster Controller has not yet set OwnerRef") + return ctrl.Result{}, nil + } + + if annotations.IsPaused(cluster, gcpCluster) { + log.Info("GCPManagedCluster or linked Cluster is marked as paused. Won't reconcile") + return ctrl.Result{}, nil + } + + log = log.WithValues("cluster", cluster.Name) + + controlPlane := &infrav1exp.GCPManagedControlPlane{} + controlPlaneRef := types.NamespacedName{ + Name: cluster.Spec.ControlPlaneRef.Name, + Namespace: cluster.Spec.ControlPlaneRef.Namespace, + } + + log.V(4).Info("getting control plane %s", controlPlaneRef) + if err := r.Get(ctx, controlPlaneRef, controlPlane); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to get control plane ref: %w", err) + } + + clusterScope, err := scope.NewManagedClusterScope(ctx, scope.ManagedClusterScopeParams{ + Client: r.Client, + Cluster: cluster, + GCPManagedCluster: gcpCluster, + GCPManagedControlPlane: controlPlane, + }) + if err != nil { + return ctrl.Result{}, errors.Errorf("failed to create scope: %+v", err) + } + + // Always close the scope when exiting this function so we can persist any GCPMachine changes. + defer func() { + if err := clusterScope.Close(); err != nil && reterr == nil { + reterr = err + } + }() + + // Handle deleted clusters + if !gcpCluster.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, clusterScope) + } + + // Handle non-deleted clusters + return r.reconcile(ctx, clusterScope) } // SetupWithManager sets up the controller with the Manager. @@ -88,6 +165,85 @@ func (r *GCPManagedClusterReconciler) SetupWithManager(ctx context.Context, mgr return nil } +func (r *GCPManagedClusterReconciler) reconcile(ctx context.Context, clusterScope *scope.ManagedClusterScope) (ctrl.Result, error) { + log := log.FromContext(ctx) + log.Info("Reconciling GCPManagedCluster") + + controllerutil.AddFinalizer(clusterScope.GCPManagedCluster, infrav1exp.ClusterFinalizer) + if err := clusterScope.PatchObject(); err != nil { + return ctrl.Result{}, err + } + + region, err := clusterScope.Cloud().Regions().Get(ctx, meta.GlobalKey(clusterScope.Region())) + if err != nil { + return ctrl.Result{}, err + } + + zones, err := clusterScope.Cloud().Zones().List(ctx, filter.Regexp("region", region.SelfLink)) + if err != nil { + return ctrl.Result{}, err + } + + failureDomains := make(clusterv1.FailureDomains, len(zones)) + for _, zone := range zones { + failureDomains[zone.Name] = clusterv1.FailureDomainSpec{ + ControlPlane: false, + } + } + clusterScope.SetFailureDomains(failureDomains) + + reconcilers := []cloud.Reconciler{ + networks.New(clusterScope), + subnets.New(clusterScope), + } + + for _, r := range reconcilers { + if err := r.Reconcile(ctx); err != nil { + log.Error(err, "Reconcile error") + record.Warnf(clusterScope.GCPManagedCluster, "GCPManagedClusterReconcile", "Reconcile error - %v", err) + return ctrl.Result{}, err + } + } + + clusterScope.SetReady() + record.Event(clusterScope.GCPManagedCluster, "GCPManagedClusterReconcile", "Ready") + + controlPlaneEndpoint := clusterScope.GCPManagedControlPlane.Spec.Endpoint + if controlPlaneEndpoint.IsZero() { + log.Info("GCPManagedControlplane does not have endpoint yet. Reconciling") + record.Event(clusterScope.GCPManagedCluster, "GCPManagedClusterReconcile", "Waiting for control-plane endpoint") + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + + clusterScope.SetControlPlaneEndpoint(controlPlaneEndpoint) + record.Eventf(clusterScope.GCPManagedCluster, "GCPManagedClusterReconcile", "Got control-plane endpoint - %s", controlPlaneEndpoint.Host) + + return ctrl.Result{}, nil +} + +func (r *GCPManagedClusterReconciler) reconcileDelete(ctx context.Context, clusterScope *scope.ManagedClusterScope) (ctrl.Result, error) { + log := log.FromContext(ctx) + log.Info("Reconciling Delete GCPManagedCluster") + + reconcilers := []cloud.Reconciler{ + subnets.New(clusterScope), + networks.New(clusterScope), + } + + for _, r := range reconcilers { + if err := r.Delete(ctx); err != nil { + log.Error(err, "Reconcile error") + record.Warnf(clusterScope.GCPManagedCluster, "GCPManagedClusterReconcile", "Reconcile error - %v", err) + return ctrl.Result{}, err + } + } + + controllerutil.RemoveFinalizer(clusterScope.GCPManagedCluster, infrav1exp.ClusterFinalizer) + record.Event(clusterScope.GCPManagedCluster, "GCPClusterReconcile", "Reconciled") + + return ctrl.Result{}, nil +} + func (r *GCPManagedClusterReconciler) managedControlPlaneMapper(ctx context.Context) handler.MapFunc { return func(o client.Object) []ctrl.Request { log := ctrl.LoggerFrom(ctx) @@ -105,7 +261,7 @@ func (r *GCPManagedClusterReconciler) managedControlPlaneMapper(ctx context.Cont } if gcpManagedControlPlane.Spec.Endpoint.IsZero() { - log.V(2).Info("GCPManagedControlPlane has no endpoint, skipping mapping") + log.V(2).Info("GCPManagedControlPlane has no endpoint, skipping mapping") return nil } diff --git a/feature/feature.go b/feature/feature.go index b5e5afb5cc..419aab8938 100644 --- a/feature/feature.go +++ b/feature/feature.go @@ -20,12 +20,16 @@ import ( ) const ( -// Every capg-specific feature gate should add method here following this template: -// -// // owner: @username -// // alpha: v1.X -// MyFeature featuregate.Feature = "MyFeature". + // Every capg-specific feature gate should add method here following this template: + // + // // owner: @username + // // alpha: v1.X + // MyFeature featuregate.Feature = "MyFeature". + // GKE is used to enable GKE support + // owner: @richardchen331 & @richardcase + // alpha: v0.1 + GKE featuregate.Feature = "GKE" ) func init() { @@ -35,6 +39,5 @@ func init() { // defaultCAPGFeatureGates consists of all known capg-specific feature keys. // To add a new feature, define a key for it above and add it here. var defaultCAPGFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - // Every feature should be initiated here: - + GKE: {Default: false, PreRelease: featuregate.Alpha}, } diff --git a/main.go b/main.go index 473c9f9b85..0183d0420c 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ limitations under the License. package main import ( + "context" "flag" "fmt" "net/http" @@ -36,6 +37,8 @@ import ( infrav1alpha4 "sigs.k8s.io/cluster-api-provider-gcp/api/v1alpha4" infrav1beta1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-gcp/controllers" + infrav1exp "sigs.k8s.io/cluster-api-provider-gcp/exp/api/v1beta1" + expcontrollers "sigs.k8s.io/cluster-api-provider-gcp/exp/controllers" "sigs.k8s.io/cluster-api-provider-gcp/feature" "sigs.k8s.io/cluster-api-provider-gcp/util/reconciler" "sigs.k8s.io/cluster-api-provider-gcp/version" @@ -58,6 +61,7 @@ func init() { _ = infrav1alpha4.AddToScheme(scheme) _ = infrav1beta1.AddToScheme(scheme) _ = clusterv1.AddToScheme(scheme) + _ = infrav1exp.AddToScheme(scheme) // +kubebuilder:scaffold:scheme } @@ -106,6 +110,8 @@ func main() { ctrl.SetLogger(klogr.New()) + setupLog.Info(fmt.Sprintf("feature gates: %+v\n", feature.Gates)) + // Machine and cluster operations can create enough events to trigger the event recorder spam filter // Setting the burst size higher ensures all events will be recorded and submitted to the API broadcaster := cgrecord.NewBroadcasterWithCorrelatorOptions(cgrecord.CorrelatorOptions{ @@ -139,56 +145,99 @@ func main() { // Setup the context that's going to be used in controllers and for the manager. ctx := ctrl.SetupSignalHandler() - if err = (&controllers.GCPMachineReconciler{ + if setupErr := setupReconcilers(ctx, mgr); setupErr != nil { + setupLog.Error(err, "unable to setup reconcilers") + os.Exit(1) + } + + if setupErr := setupWebhooks(mgr); setupErr != nil { + setupLog.Error(err, "unable to setup webhooks") + os.Exit(1) + } + + if setupErr := setupProbes(mgr); setupErr != nil { + setupLog.Error(err, "unable to setup probes") + os.Exit(1) + } + + // +kubebuilder:scaffold:builder + setupLog.Info("starting manager", "version", version.Get().String(), "extended_info", version.Get()) + if err := mgr.Start(ctx); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} + +func setupReconcilers(ctx context.Context, mgr ctrl.Manager) error { + if err := (&controllers.GCPMachineReconciler{ Client: mgr.GetClient(), ReconcileTimeout: reconcileTimeout, WatchFilterValue: watchFilterValue, }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: gcpMachineConcurrency}); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "GCPMachine") - os.Exit(1) + return fmt.Errorf("setting up GCPMachine controller: %w", err) } - if err = (&controllers.GCPClusterReconciler{ + if err := (&controllers.GCPClusterReconciler{ Client: mgr.GetClient(), ReconcileTimeout: reconcileTimeout, WatchFilterValue: watchFilterValue, }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: gcpClusterConcurrency}); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "GCPCluster") - os.Exit(1) + return fmt.Errorf("setting up GCPCluster controller: %w", err) } - if err = (&infrav1beta1.GCPCluster{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "GCPCluster") - os.Exit(1) + if feature.Gates.Enabled(feature.GKE) { + if err := (&expcontrollers.GCPManagedClusterReconciler{ + Client: mgr.GetClient(), + ReconcileTimeout: reconcileTimeout, + WatchFilterValue: watchFilterValue, + }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: gcpClusterConcurrency}); err != nil { + return fmt.Errorf("setting up GCPManagedCluster controller: %w", err) + } + + //TODO: add other controllers } - if err = (&infrav1beta1.GCPClusterTemplate{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "GCPClusterTemplate") - os.Exit(1) + + return nil +} + +func setupWebhooks(mgr ctrl.Manager) error { + if err := (&infrav1beta1.GCPCluster{}).SetupWebhookWithManager(mgr); err != nil { + return fmt.Errorf("setting up GCPCluster webhook: %w", err) } - if err = (&infrav1beta1.GCPMachine{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "GCPMachine") - os.Exit(1) + if err := (&infrav1beta1.GCPClusterTemplate{}).SetupWebhookWithManager(mgr); err != nil { + return fmt.Errorf("setting up GCPClusterTemplate webhook: %w", err) } - if err = (&infrav1beta1.GCPMachineTemplate{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "GCPMachineTemplate") - os.Exit(1) + if err := (&infrav1beta1.GCPMachine{}).SetupWebhookWithManager(mgr); err != nil { + return fmt.Errorf("setting up GCPMachine webhook: %w", err) + } + if err := (&infrav1beta1.GCPMachineTemplate{}).SetupWebhookWithManager(mgr); err != nil { + return fmt.Errorf("setting up GCPMachineTemplate webhook: %w", err) + } + + if feature.Gates.Enabled(feature.GKE) { + if err := (&infrav1exp.GCPManagedCluster{}).SetupWebhookWithManager(mgr); err != nil { + return fmt.Errorf("setting up GCPManagedCluster webhook: %w", err) + } + if err := (&infrav1exp.GCPManagedControlPlane{}).SetupWebhookWithManager(mgr); err != nil { + return fmt.Errorf("setting up GCPManagedControlPlane webhook: %w", err) + } + if err := (&infrav1exp.GCPManagedMachinePool{}).SetupWebhookWithManager(mgr); err != nil { + return fmt.Errorf("setting up GCPManagedMachinePool webhook: %w", err) + } } + return nil +} + +func setupProbes(mgr ctrl.Manager) error { if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { - setupLog.Error(err, "unable to create ready check") - os.Exit(1) + return fmt.Errorf("creating ready check: %w", err) } if err := mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { - setupLog.Error(err, "unable to create health check") - os.Exit(1) + return fmt.Errorf("creating health check: %w", err) } - // +kubebuilder:scaffold:builder - setupLog.Info("starting manager", "version", version.Get().String(), "extended_info", version.Get()) - if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } + return nil } func initFlags(fs *pflag.FlagSet) {