Skip to content

Commit

Permalink
Merge pull request openshift#660 from sjenning/hostedcluster-validation
Browse files Browse the repository at this point in the history 
Add SupportedHostedCluster condition HostedCluster
  • Loading branch information
@openshift-merge-robot
openshift-merge-robot authored Nov 10, 2021
2 parents b2f2b1c + e25e123 commit 0bb7d4f
Show file tree
Hide file tree
Showing 6 changed files with 171 additions and 77 deletions.
9 changes: 9 additions & 0 deletions api/v1alpha1/hostedcluster_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -740,6 +740,13 @@ const (
// ValidHostedClusterConfiguration indicates (if status is true) that the
// ClusterConfiguration specified for the HostedCluster is valid.
ValidHostedClusterConfiguration ConditionType = "ValidConfiguration"

// SupportedHostedCluster indicates whether a HostedCluster is supported by
// the current configuration of the hypershift-operator.
// e.g. If HostedCluster requests endpointAcess Private but the hypershift-operator
// is running on a management cluster outside AWS or is not configured with AWS
// credentials, the HostedCluster is not supported.
SupportedHostedCluster ConditionType = "SupportedHostedCluster"
)

const (
Expand All @@ -752,6 +759,8 @@ const (
HostedClusterUnhealthyComponentsReason = "UnhealthyControlPlaneComponents"
InvalidConfigurationReason = "InvalidConfiguration"

UnsupportedHostedClusterReason = "UnsupportedHostedCluster"

UnmanagedEtcdStatusUnknownReason = "UnmanagedEtcdStatusUnknown"
UnmanagedEtcdMisconfiguredReason = "UnmanagedEtcdMisconfigured"
UnmanagedEtcdAsExpected = "UnmanagedEtcdAsExpected"
Expand Down
69 changes: 42 additions & 27 deletions cmd/install/assets/hypershift_operator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,12 @@ func (o HyperShiftNamespace) Build() *corev1.Namespace {
return namespace
}

type HyperShiftOperatorAWSCredentialsSecret struct {
Namespace *corev1.Namespace
AWSCredsBytes []byte
type HyperShiftOperatorCredentialsSecret struct {
Namespace *corev1.Namespace
CredsBytes []byte
}

func (o HyperShiftOperatorAWSCredentialsSecret) Build() *corev1.Secret {
func (o HyperShiftOperatorCredentialsSecret) Build() *corev1.Secret {
secret := &corev1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
Expand All @@ -53,7 +53,7 @@ func (o HyperShiftOperatorAWSCredentialsSecret) Build() *corev1.Secret {
Namespace: o.Namespace.Name,
},
Data: map[string][]byte{
"credentials": o.AWSCredsBytes,
"credentials": o.CredsBytes,
},
}
return secret
Expand All @@ -66,7 +66,9 @@ type HyperShiftOperatorDeployment struct {
Replicas int32
EnableOCPClusterMonitoring bool
EnableCIDebugOutput bool
AWSRegion string
PrivatePlatform string
AWSPrivateCreds string
AWSPrivateRegion string
}

func (o HyperShiftOperatorDeployment) Build() *appsv1.Deployment {
Expand Down Expand Up @@ -114,42 +116,55 @@ func (o HyperShiftOperatorDeployment) Build() *appsv1.Deployment {
},
},
},
{
Name: "AWS_SHARED_CREDENTIALS_FILE",
Value: "/etc/aws/credentials",
},
},
Command: []string{"/usr/bin/hypershift-operator"},
Args: []string{"run", "--namespace=$(MY_NAMESPACE)", "--deployment-name=operator", "--metrics-addr=:9000", fmt.Sprintf("--enable-ocp-cluster-monitoring=%t", o.EnableOCPClusterMonitoring), fmt.Sprintf("--enable-ci-debug-output=%t", o.EnableCIDebugOutput), fmt.Sprintf("--aws-region=%s", o.AWSRegion)},
Args: []string{"run", "--namespace=$(MY_NAMESPACE)", "--deployment-name=operator", "--metrics-addr=:9000", fmt.Sprintf("--enable-ocp-cluster-monitoring=%t", o.EnableOCPClusterMonitoring), fmt.Sprintf("--enable-ci-debug-output=%t", o.EnableCIDebugOutput)},
Ports: []corev1.ContainerPort{
{
Name: "metrics",
ContainerPort: 9000,
Protocol: corev1.ProtocolTCP,
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "credentials",
MountPath: "/etc/aws",
},
},
},
},
Volumes: []corev1.Volume{
{
Name: "credentials",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "hypershift-operator-creds",
},
},
},
},
},
},
},
}

privatePlatformType := hyperv1.PlatformType(o.PrivatePlatform)
if privatePlatformType == hyperv1.NonePlatform {
return deployment
}

// Add generic provider credentials secret volume
deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "credentials",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "hypershift-operator-creds",
},
},
})
deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "credentials",
MountPath: "/etc/provider",
})

// Add platform specific settings
switch privatePlatformType {
case hyperv1.AWSPlatform:
deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env,
corev1.EnvVar{
Name: "AWS_SHARED_CREDENTIALS_FILE",
Value: "/etc/provider/credentials",
},
corev1.EnvVar{
Name: "AWS_REGION",
Value: o.AWSPrivateRegion,
})
}
return deployment
}

Expand Down
76 changes: 52 additions & 24 deletions cmd/install/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"k8s.io/apimachinery/pkg/types"

hyperapi "github.com/openshift/hypershift/api"
hyperv1 "github.com/openshift/hypershift/api/v1alpha1"
"github.com/openshift/hypershift/cmd/install/assets"
"github.com/openshift/hypershift/cmd/util"
"github.com/openshift/hypershift/cmd/version"
Expand All @@ -46,8 +47,22 @@ type Options struct {
ExcludeEtcdManifests bool
EnableOCPClusterMonitoring bool
EnableCIDebugOutput bool
AWSCreds string
AWSRegion string
PrivatePlatform string
AWSPrivateCreds string
AWSPrivateRegion string
}

func (o *Options) Validate() error {
switch hyperv1.PlatformType(o.PrivatePlatform) {
case hyperv1.AWSPlatform:
if o.AWSPrivateCreds == "" || o.AWSPrivateRegion == "" {
return fmt.Errorf("--aws-private-region and --aws-private-creds are required with --private-platform=%s", hyperv1.AWSPlatform)
}
case hyperv1.NonePlatform:
default:
return fmt.Errorf("--private-platform must be either %s or %s", hyperv1.AWSPlatform, hyperv1.NonePlatform)
}
return nil
}

func NewCommand() *cobra.Command {
Expand All @@ -62,7 +77,7 @@ func NewCommand() *cobra.Command {
opts.EnableOCPClusterMonitoring = true
opts.EnableCIDebugOutput = true
}
opts.AWSRegion = "us-east-1"
opts.PrivatePlatform = string(hyperv1.NonePlatform)

cmd.Flags().StringVar(&opts.Namespace, "namespace", "hypershift", "The namespace in which to install HyperShift")
cmd.Flags().StringVar(&opts.HyperShiftImage, "hypershift-image", version.HyperShiftImage, "The HyperShift image to deploy")
Expand All @@ -71,10 +86,15 @@ func NewCommand() *cobra.Command {
cmd.Flags().BoolVar(&opts.ExcludeEtcdManifests, "exclude-etcd", false, "Leave out etcd manifests")
cmd.Flags().BoolVar(&opts.EnableOCPClusterMonitoring, "enable-ocp-cluster-monitoring", opts.EnableOCPClusterMonitoring, "Development-only option that will make your OCP cluster unsupported: If the cluster Prometheus should be configured to scrape metrics")
cmd.Flags().BoolVar(&opts.EnableCIDebugOutput, "enable-ci-debug-output", opts.EnableCIDebugOutput, "If extra CI debug output should be enabled")
cmd.Flags().StringVar(&opts.AWSCreds, "aws-creds", opts.AWSCreds, "Path to an AWS credentials file")
cmd.Flags().StringVar(&opts.AWSRegion, "aws-region", opts.AWSRegion, "AWS region in which the operator will create resources")
cmd.Flags().StringVar(&opts.PrivatePlatform, "private-platform", opts.PrivatePlatform, "Platform on which private clusters are supported by this operator (supports \"AWS\" or \"None\")")
cmd.Flags().StringVar(&opts.AWSPrivateCreds, "aws-private-creds", opts.AWSPrivateCreds, "Path to an AWS credentials file with privileges sufficient to manage private cluster resources")
cmd.Flags().StringVar(&opts.AWSPrivateRegion, "aws-private-region", opts.AWSPrivateRegion, "AWS region where private clusters are supported by this operator")

cmd.RunE = func(cmd *cobra.Command, args []string) error {
if err := opts.Validate(); err != nil {
return err
}

cmd.Run = func(cmd *cobra.Command, args []string) {
ctx, cancel := context.WithCancel(context.Background())
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT)
Expand All @@ -90,17 +110,21 @@ func NewCommand() *cobra.Command {
opts.HyperShiftOperatorReplicas = 1
}

objects := hyperShiftOperatorManifests(opts)
objects, err := hyperShiftOperatorManifests(opts)
if err != nil {
return err
}

switch {
case opts.Render:
render(objects)
default:
err := apply(ctx, objects)
if err != nil {
panic(err)
return err
}
}
return nil
}

return cmd
Expand Down Expand Up @@ -145,7 +169,7 @@ func apply(ctx context.Context, objects []crclient.Object) error {
return nil
}

func hyperShiftOperatorManifests(opts Options) []crclient.Object {
func hyperShiftOperatorManifests(opts Options) ([]crclient.Object, error) {
controlPlanePriorityClass := assets.HyperShiftControlPlanePriorityClass{}.Build()
etcdPriorityClass := assets.HyperShiftEtcdPriorityClass{}.Build()
apiCriticalPriorityClass := assets.HyperShiftAPICriticalPriorityClass{}.Build()
Expand All @@ -168,26 +192,16 @@ func hyperShiftOperatorManifests(opts Options) []crclient.Object {
ServiceAccount: operatorServiceAccount,
Role: operatorRole,
}.Build()
var awsCredBytes []byte
if len(opts.AWSCreds) != 0 {
var err error
awsCredBytes, err = ioutil.ReadFile(opts.AWSCreds)
if err != nil {
panic(err)
}
}
operatorAWSCredentialsSecret := assets.HyperShiftOperatorAWSCredentialsSecret{
Namespace: operatorNamespace,
AWSCredsBytes: awsCredBytes,
}.Build()
operatorDeployment := assets.HyperShiftOperatorDeployment{
Namespace: operatorNamespace,
OperatorImage: opts.HyperShiftImage,
ServiceAccount: operatorServiceAccount,
Replicas: opts.HyperShiftOperatorReplicas,
EnableOCPClusterMonitoring: opts.EnableOCPClusterMonitoring,
EnableCIDebugOutput: opts.EnableCIDebugOutput,
AWSRegion: opts.AWSRegion,
PrivatePlatform: opts.PrivatePlatform,
AWSPrivateRegion: opts.AWSPrivateRegion,
AWSPrivateCreds: opts.AWSPrivateCreds,
}.Build()
operatorService := assets.HyperShiftOperatorService{
Namespace: operatorNamespace,
Expand All @@ -209,6 +223,20 @@ func hyperShiftOperatorManifests(opts Options) []crclient.Object {

var objects []crclient.Object

var credBytes []byte
switch hyperv1.PlatformType(opts.PrivatePlatform) {
case hyperv1.AWSPlatform:
var err error
credBytes, err = ioutil.ReadFile(opts.AWSPrivateCreds)
if err != nil {
return objects, err
}
}
operatorCredentialsSecret := assets.HyperShiftOperatorCredentialsSecret{
Namespace: operatorNamespace,
CredsBytes: credBytes,
}.Build()

objects = append(objects, assets.CustomResourceDefinitions(func(path string) bool {
if strings.Contains(path, "etcd") && opts.ExcludeEtcdManifests {
return false
Expand All @@ -225,13 +253,13 @@ func hyperShiftOperatorManifests(opts Options) []crclient.Object {
objects = append(objects, operatorClusterRoleBinding)
objects = append(objects, operatorRole)
objects = append(objects, operatorRoleBinding)
objects = append(objects, operatorAWSCredentialsSecret)
objects = append(objects, operatorCredentialsSecret)
objects = append(objects, operatorDeployment)
objects = append(objects, operatorService)
objects = append(objects, prometheusRole)
objects = append(objects, prometheusRoleBinding)
objects = append(objects, serviceMonitor)
objects = append(objects, recordingRule)

return objects
return objects, nil
}
50 changes: 50 additions & 0 deletions hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"crypto/x509/pkix"
"fmt"
"net"
"os"
"strings"
"time"

Expand Down Expand Up @@ -135,6 +136,8 @@ type HostedClusterReconciler struct {
upsert.CreateOrUpdateProvider

EnableCIDebugOutput bool

PrivatePlatform hyperv1.PlatformType
}

// +kubebuilder:rbac:groups=hypershift.openshift.io,resources=hostedclusters,verbs=get;list;watch;create;update;patch;delete
Expand Down Expand Up @@ -312,6 +315,24 @@ func (r *HostedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reques
meta.SetStatusCondition(&hcluster.Status.Conditions, condition)
}

// Set SupportedHostedCluster condition
{
condition := metav1.Condition{
Type: string(hyperv1.SupportedHostedCluster),
ObservedGeneration: hcluster.Generation,
}
if err := r.validateHostedClusterSupport(hcluster, r.PrivatePlatform); err != nil {
condition.Status = metav1.ConditionFalse
condition.Message = err.Error()
condition.Reason = hyperv1.UnsupportedHostedClusterReason
} else {
condition.Status = metav1.ConditionTrue
condition.Message = "HostedCluster is support by operator configuration"
condition.Reason = hyperv1.HostedClusterAsExpectedReason
}
meta.SetStatusCondition(&hcluster.Status.Conditions, condition)
}

// Set ValidHostedControlPlaneConfiguration condition
{
controlPlaneNamespace := manifests.HostedControlPlaneNamespace(hcluster.Namespace, hcluster.Name)
Expand Down Expand Up @@ -457,6 +478,11 @@ func (r *HostedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reques
r.Log.Info("Configuration is invalid, reconciliation is blocked")
return ctrl.Result{}, nil
}
supportedHostedCluster := meta.FindStatusCondition(hcluster.Status.Conditions, string(hyperv1.SupportedHostedCluster))
if supportedHostedCluster != nil && supportedHostedCluster.Status == metav1.ConditionFalse {
r.Log.Info("Hosted Cluster is not supported by operator configuration, reconciliation is blocked")
return ctrl.Result{}, nil
}
}

// Reconcile the hosted cluster namespace
Expand Down Expand Up @@ -3004,6 +3030,30 @@ func (r *HostedClusterReconciler) validateConfigAndClusterCapabilities(hc *hyper
return nil
}

func (r *HostedClusterReconciler) validateHostedClusterSupport(hc *hyperv1.HostedCluster, privatePlatform hyperv1.PlatformType) error {
switch hc.Spec.Platform.Type {
case hyperv1.AWSPlatform:
if hc.Spec.Platform.AWS == nil {
return nil
}
if hc.Spec.Platform.AWS.EndpointAccess == hyperv1.Public {
return nil
}
region := os.Getenv("AWS_REGION")
if region == "" {
return fmt.Errorf("AWS_REGION environment variable is not set for the operator")
}
credFile := os.Getenv("AWS_SHARED_CREDENTIALS_FILE")
if credFile == "" {
return fmt.Errorf("AWS_SHARED_CREDENTIALS_FILE environment variable is not set for the operator")
}
if hc.Spec.Platform.AWS.Region != region {
return fmt.Errorf("operator only supports private clusters in region %s", region)
}
}
return nil
}

type ClusterMachineApproverConfig struct {
NodeClientCert NodeClientCert `json:"nodeClientCert,omitempty"`
}
Expand Down
Loading

0 comments on commit 0bb7d4f

Please sign in to comment.