Skip to content

Commit

Permalink
Webhooks for OSM (#106)
Browse files Browse the repository at this point in the history
* Add mutating webhook for OSP and MD

* Add validation for OSP

Signed-off-by: Waleed Malik <[email protected]>

* Add validations in OSP

* Add vanilla manifests for deployment OSM and webhook

* Bump GO to v1.17.6

* Bump GO to v1.17.5

Signed-off-by: Waleed Malik <[email protected]>

* Use quay.io for images

* Add deployment for osm-controller and webhook

* Use dedicated client for interactions with seed and user clusters

* Refactored code

* Update test data

* Update generated code

Signed-off-by: Waleed Malik <[email protected]>

* test changes

* Fix codegen

* Fix path for tls cert

* Fix deployments and use external client for osc controller

* Remove dependencies from ValidateMachineDeployment method to make it re-usable

* Refactored code; handle PR feedback

* User worker manager for OSC controller

* Handle PR feedback
  • Loading branch information
ahmedwaleedmalik authored Dec 21, 2021
1 parent 3ded3fb commit dcbfc27
Show file tree
Hide file tree
Showing 43 changed files with 1,746 additions and 366 deletions.
7 changes: 5 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

ARG GO_VERSION=1.17.5
FROM golang:${GO_VERSION} AS builder
WORKDIR /go/src/github.com/kubermatic/operating-system-manager
WORKDIR /go/src/k8c.io/operating-system-manager
COPY . .
RUN make all

Expand All @@ -22,6 +23,8 @@ FROM alpine:3.12
RUN apk add --no-cache ca-certificates cdrkit

COPY --from=builder \
/go/src/github.com/kubermatic/operating-system-manager/_build/osm-controller \
/go/src/k8c.io/operating-system-manager/_build/osm-controller \
/go/src/k8c.io/operating-system-manager/_build/webhook \
/usr/local/bin/

USER nobody
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ Simply run `make test`
To run OSM locally:

- Either use a [kind](https://kind.sigs.k8s.io/docs/user/quick-start/) cluster or actual cluster and make sure that the correct context is loaded
- Run `kubectl apply -f charts/crd` to install CRDs
- Run `kubectl apply -f deploy/crds` to install CRDs
- Create relevant OperatingSystemProfile resources. Check [sample](./examples) for reference.
- Run `make run`

Expand Down
131 changes: 107 additions & 24 deletions cmd/osm-controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,20 @@ import (
clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1"
"k8c.io/operating-system-manager/pkg/controllers/osc"
"k8c.io/operating-system-manager/pkg/controllers/osp"
"k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1"
osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1"
"k8c.io/operating-system-manager/pkg/generator"
providerconfig "k8c.io/operating-system-manager/pkg/providerconfig/config"

utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes/scheme"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"k8s.io/klog"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
)

type options struct {
Expand All @@ -54,8 +59,24 @@ type options struct {
nodePortRange string
podCidr string

clusterDNSIPs string
kubeconfig string
clusterDNSIPs string
workerClusterKubeconfig string
kubeconfig string

healthProbeAddress string
metricsAddress string
workerHealthProbeAddress string
workerMetricsAddress string
}

const (
defaultLeaderElectionNamespace = "kube-system"
)

func init() {
utilruntime.Must(clientgoscheme.AddToScheme(scheme.Scheme))
utilruntime.Must(osmv1alpha1.AddToScheme(scheme.Scheme))
utilruntime.Must(clusterv1alpha1.AddToScheme(scheme.Scheme))
}

func main() {
Expand All @@ -66,7 +87,7 @@ func main() {
if flag.Lookup("kubeconfig") == nil {
flag.StringVar(&opt.kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
}

flag.StringVar(&opt.workerClusterKubeconfig, "worker-cluster-kubeconfig", "", "Path to kubeconfig of cluster where provisioning secrets are created")
flag.IntVar(&opt.workerCount, "worker-count", 10, "Number of workers which process reconciliation in parallel.")
flag.StringVar(&opt.clusterName, "cluster-name", "", "The cluster where the OSC will run.")
flag.StringVar(&opt.namespace, "namespace", "", "The namespace where the OSC controller will run.")
Expand All @@ -80,6 +101,11 @@ func main() {
flag.StringVar(&opt.podCidr, "pod-cidr", "172.25.0.0/16", "The network ranges from which POD networks are allocated")
flag.StringVar(&opt.nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility")

flag.StringVar(&opt.healthProbeAddress, "health-probe-address", "127.0.0.1:8085", "The address on which the liveness check on /healthz and readiness check on /readyz will be available")
flag.StringVar(&opt.metricsAddress, "metrics-address", "127.0.0.1:8080", "The address on which Prometheus metrics will be available under /metrics")

flag.StringVar(&opt.workerHealthProbeAddress, "worker-health-probe-address", "127.0.0.1:8086", "For worker manager, the address on which the liveness check on /healthz and readiness check on /readyz will be available")
flag.StringVar(&opt.workerMetricsAddress, "worker-metrics-address", "127.0.0.1:8081", "For worker manager, the address on which Prometheus metrics will be available under /metrics")
flag.Parse()

if len(opt.namespace) == 0 {
Expand All @@ -102,40 +128,70 @@ func main() {
klog.Fatalf("invalid cluster dns specified: %v", err)
}

mgr, err := manager.New(config.GetConfigOrDie(), manager.Options{})
logger, err := zap.NewProduction()
if err != nil {
klog.Error(err, "could not create manager")
os.Exit(1)
}
if err = v1alpha1.AddToScheme(mgr.GetScheme()); err != nil {
klog.Fatal(err)
}
// because we watch MachineDeployments
if err = clusterv1alpha1.AddToScheme(mgr.GetScheme()); err != nil {
klog.Fatal(err)
}
logger, err := zap.NewProduction()
log := logger.Sugar()

// Create manager with client against in-cluster config
mgr, err := createManager(opt)
if err != nil {
klog.Fatal(err)
klog.Fatalf("failed to create runtime manager: %v", err)
}

log := logger.Sugar()
// Start with assuming that current cluster will be used as worker cluster
workerMgr := mgr

if err := osp.Add(mgr, log, opt.namespace, opt.workerCount); err != nil {
klog.Fatal(err)
// Handling for worker cluster
if opt.workerClusterKubeconfig != "" {
workerClusterConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: opt.workerClusterKubeconfig},
&clientcmd.ConfigOverrides{}).ClientConfig()
if err != nil {
klog.Fatal(err)
}

workerMgr, err = manager.New(workerClusterConfig, manager.Options{
LeaderElection: true,
LeaderElectionID: "operating-system-manager-worker-manager",
LeaderElectionNamespace: defaultLeaderElectionNamespace,
HealthProbeBindAddress: opt.workerHealthProbeAddress,
MetricsBindAddress: opt.workerMetricsAddress,
Port: 9444,
})
if err != nil {
klog.Fatal(err)
}

// "-worker-cluster-kubeconfig" was not empty and a valid kubeconfig was provided,
// point workerClient to the external cluster
// Use workerClusterKubeconfig since the machines will exist on that cluster
opt.kubeconfig = opt.workerClusterKubeconfig

if err := mgr.Add(workerMgr); err != nil {
klog.Fatal("failed to add workers cluster mgr to main mgr", zap.Error(err))
}
}

// Instantiate ConfigVarResolver
providerconfig.SetConfigVarResolver(context.Background(), mgr.GetClient(), opt.namespace)
providerconfig.SetConfigVarResolver(context.Background(), workerMgr.GetClient(), opt.namespace)

// Setup OSP controller
if err := osp.Add(mgr, log, opt.namespace, opt.workerCount); err != nil {
klog.Fatal(err)
}

// Setup OSC controller
if err := osc.Add(
mgr,
workerMgr,
log,
osc.CloudInitSettingsNamespace,
mgr.GetClient(),
opt.kubeconfig,
opt.namespace,
opt.clusterName,
opt.workerCount,
parsedClusterDNSIPs,
opt.kubeconfig,
generator.NewDefaultCloudConfigGenerator(""),
opt.containerRuntime,
opt.externalCloudProvider,
Expand All @@ -149,11 +205,38 @@ func main() {
klog.Fatal(err)
}

if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
log.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
klog.Fatalf("Failed to start OSC controller: %v", zap.Error(err))
}
}

func createManager(opt *options) (manager.Manager, error) {
// Manager options
options := manager.Options{
LeaderElection: true,
LeaderElectionID: "operating-system-manager",
LeaderElectionNamespace: defaultLeaderElectionNamespace,
HealthProbeBindAddress: opt.healthProbeAddress,
MetricsBindAddress: opt.metricsAddress,
Port: 9443,
}

mgr, err := manager.New(config.GetConfigOrDie(), options)
if err != nil {
return nil, fmt.Errorf("error building ctrlruntime manager: %v", err)
}

// Add health endpoints
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
return nil, fmt.Errorf("failed to add health check: %v", err)
}
if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
return nil, fmt.Errorf("failed to add readiness check: %v", err)
}
return mgr, nil
}

func parseClusterDNSIPs(s string) ([]net.IP, error) {
var ips []net.IP
sips := strings.Split(s, ",")
Expand Down
96 changes: 96 additions & 0 deletions cmd/webhook/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
/*
Copyright 2021 The Operating System Manager contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"flag"

clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1"
"k8c.io/operating-system-manager/pkg/admission"
"k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1"

"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/klog"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
)

type options struct {
namespace string

admissionListenAddress string
admissionTLSCertPath string
admissionTLSKeyPath string
}

var (
scheme = runtime.NewScheme()
)

func init() {
utilruntime.Must(v1alpha1.AddToScheme(scheme))
utilruntime.Must(clusterv1alpha1.AddToScheme(scheme))
}

func main() {
klog.InitFlags(nil)

opt := &options{}

flag.StringVar(&opt.admissionListenAddress, "listen-address", ":9876", "The address on which the MutatingWebhook will listen on")
flag.StringVar(&opt.admissionTLSCertPath, "tls-cert-path", "/tmp/cert/cert.pem", "The path of the TLS cert for the MutatingWebhook")
flag.StringVar(&opt.admissionTLSKeyPath, "tls-key-path", "/tmp/cert/key.pem", "The path of the TLS key for the MutatingWebhook")
flag.StringVar(&opt.namespace, "namespace", "", "The namespace where the OSC webhook will run.")
flag.Parse()

if len(opt.namespace) == 0 {
klog.Fatal("-namespace is required")
}

// Build config for in-cluster cluster
cfg, err := config.GetConfig()
if err != nil {
klog.Fatalf("error building kubeconfig: %v", err)
}

// Build client against in-cluster config
client, err := ctrlruntimeclient.New(cfg, ctrlruntimeclient.Options{
Scheme: scheme,
})
if err != nil {
klog.Fatalf("failed to build seed client: %v", err)
}

srv, err := admission.New(opt.admissionListenAddress, opt.namespace, client)
if err != nil {
klog.Fatalf("failed to create admission hook: %v", err)
}

klog.Infof("starting webhook server on %s", opt.admissionListenAddress)

if err := srv.ListenAndServeTLS(opt.admissionTLSCertPath, opt.admissionTLSKeyPath); err != nil {
klog.Fatalf("failed to start server: %v", err)
}
defer func() {
if err := srv.Close(); err != nil {
klog.Fatalf("failed to shutdown server: %v", err)
}
}()
klog.Infof("Listening on %s", opt.admissionListenAddress)
select {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,23 @@ spec:
name:
description: Name represents the name of the supported cloud provider
type: string
enum:
- aws
- azure
- digitalocean
- gce
- hetzner
- kubevirt
- linode
- openstack
- packet
- vsphere
- fake
- alibaba
- anexia
- scaleway
- baremetal
- external
spec:
description: Spec represents the os/image reference in the supported cloud provider
type: object
Expand Down Expand Up @@ -121,6 +138,13 @@ spec:
osName:
description: 'OSType represent the operating system name e.g: ubuntu'
type: string
enum:
- flatcar
- rhel
- centos
- ubuntu
- sles
- amzn2
osVersion:
description: OSVersion the version of the operating system
type: string
Expand Down
Loading

0 comments on commit dcbfc27

Please sign in to comment.