Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(metrics-server): use cert-manager certs
Signed-off-by: Vaughn Dice <[email protected]>
- Loading branch information
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| apiVersion: cert-manager.io/v1 | ||
| kind: Certificate | ||
| metadata: | ||
| name: {{ include "spin-operator.fullname" . }}-metrics-certs | ||
| labels: | ||
| {{- include "spin-operator.labels" . | nindent 4 }} | ||
| spec: | ||
| dnsNames: | ||
| - '{{ include "{{ .Release.Namespace }}.fullname" . }}-metrics-service.{{ .Release.Namespace | ||
| }}.svc' | ||
| - '{{ include "{{ .Release.Namespace }}.fullname" . }}-metrics-service.{{ .Release.Namespace | ||
| }}.svc.{{ .Values.kubernetesClusterDomain }}' | ||
| issuerRef: | ||
| kind: Issuer | ||
| name: '{{ include "spin-operator.fullname" . }}-selfsigned-issuer' | ||
| secretName: metrics-server-cert |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -20,6 +20,7 @@ import ( | |
| "crypto/tls" | ||
| "flag" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) | ||
| // to ensure that exec-entrypoint and run can make use of them. | ||
|
|
@@ -31,6 +32,7 @@ import ( | |
| utilruntime "k8s.io/apimachinery/pkg/util/runtime" | ||
| clientgoscheme "k8s.io/client-go/kubernetes/scheme" | ||
| ctrl "sigs.k8s.io/controller-runtime" | ||
| "sigs.k8s.io/controller-runtime/pkg/certwatcher" | ||
| "sigs.k8s.io/controller-runtime/pkg/healthz" | ||
| "sigs.k8s.io/controller-runtime/pkg/log/zap" | ||
| "sigs.k8s.io/controller-runtime/pkg/metrics/filters" | ||
|
|
@@ -57,13 +59,17 @@ func init() { | |
| func main() { | ||
| // TODO: Migrate to using github.com/alecthomas/kong so we can improve CLI interface, use env variables etc. | ||
| var metricsAddr string | ||
| var metricsCertPath, metricsCertName, metricsCertKey string | ||
| var enableLeaderElection bool | ||
| var probeAddr string | ||
| var enableWebhooks bool | ||
| var secureMetrics bool | ||
| var enableHTTP2 bool | ||
| var tlsOpts []func(*tls.Config) | ||
| flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") | ||
| flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") | ||
| flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") | ||
| flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") | ||
| flag.StringVar(&probeAddr, "health-probe-bind-address", ":8082", "The address the probe endpoint binds to.") | ||
| flag.BoolVar(&enableLeaderElection, "leader-elect", false, | ||
| "Enable leader election for controller manager. "+ | ||
|
|
@@ -83,10 +89,15 @@ func main() { | |
|
|
||
| setupLog.Info("flag values", | ||
| "metricsAddr", metricsAddr, | ||
| "secureMetrics", secureMetrics, | ||
| "enableHTTP2", enableHTTP2, | ||
| "probeAddr", probeAddr, | ||
| "enableLeaderElection", enableLeaderElection, | ||
| "enableWebhooks", enableWebhooks) | ||
|
|
||
| // Create watcher for metrics certificates | ||
| var metricsCertWatcher *certwatcher.CertWatcher | ||
|
|
||
| // if the enable-http2 flag is false (the default), http/2 should be disabled | ||
| // due to its vulnerabilities. More specifically, disabling http/2 will | ||
| // prevent from being vulnerable to the HTTP/2 Stream Cancellation and | ||
|
|
@@ -118,10 +129,28 @@ func main() { | |
| // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: | ||
| // https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization | ||
| metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization | ||
| } | ||
|
|
||
| // If the certificate is not specified, controller-runtime will automatically | ||
| // generate self-signed certificates for the metrics server. While convenient for development and testing, | ||
| // this setup is not recommended for production. | ||
| if len(metricsCertPath) > 0 { | ||
| setupLog.Info("Initializing metrics certificate watcher using provided certificates", | ||
| "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) | ||
|
|
||
| var err error | ||
| metricsCertWatcher, err = certwatcher.New( | ||
| filepath.Join(metricsCertPath, metricsCertName), | ||
| filepath.Join(metricsCertPath, metricsCertKey), | ||
| ) | ||
| if err != nil { | ||
| setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { | ||
| config.GetCertificate = metricsCertWatcher.GetCertificate | ||
| }) | ||
| } | ||
|
|
||
| mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ | ||
|
|
@@ -147,6 +176,14 @@ func main() { | |
| os.Exit(1) | ||
| } | ||
|
|
||
| if metricsCertWatcher != nil { | ||
| setupLog.Info("Adding metrics certificate watcher to manager") | ||
| if err := mgr.Add(metricsCertWatcher); err != nil { | ||
| setupLog.Error(err, "unable to add metrics certificate watcher to manager") | ||
| os.Exit(1) | ||
| } | ||
| } | ||
|
|
||
| if err = (&controller.SpinAppReconciler{ | ||
| Client: mgr.GetClient(), | ||
| Scheme: mgr.GetScheme(), | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| # The following manifests contain a self-signed issuer CR and a metrics certificate CR. | ||
| # More document can be found at https://docs.cert-manager.io | ||
| apiVersion: cert-manager.io/v1 | ||
| kind: Certificate | ||
| metadata: | ||
| labels: | ||
| app.kubernetes.io/name: project | ||
| app.kubernetes.io/managed-by: kustomize | ||
| name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml | ||
| namespace: system | ||
| spec: | ||
| dnsNames: | ||
| # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize | ||
| # replacements in the config/default/kustomization.yaml file. | ||
| - SERVICE_NAME.SERVICE_NAMESPACE.svc | ||
| - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local | ||
| issuerRef: | ||
| kind: Issuer | ||
| name: selfsigned-issuer | ||
| secretName: metrics-server-cert | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,21 +1,3 @@ | ||
| apiVersion: cert-manager.io/v1 | ||
| kind: Certificate | ||
| metadata: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| # The following manifest contains a self-signed issuer CR. | ||
| # More information can be found at https://docs.cert-manager.io | ||
| # WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. | ||
| apiVersion: cert-manager.io/v1 | ||
| kind: Issuer | ||
| metadata: | ||
| labels: | ||
| app.kubernetes.io/name: project | ||
| app.kubernetes.io/managed-by: kustomize | ||
| name: selfsigned-issuer | ||
| namespace: system | ||
| spec: | ||
| selfSigned: {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,7 @@ | ||
| resources: | ||
| - issuer.yaml | ||
| - certificate-webhook.yaml | ||
| - certificate-metrics.yaml | ||
|
|
||
| configurations: | ||
| - kustomizeconfig.yaml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| # This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. | ||
|
|
||
| # Add the volumeMount for the metrics-server certs | ||
| - op: add | ||
| path: /spec/template/spec/containers/0/volumeMounts/- | ||
| value: | ||
| mountPath: /tmp/k8s-metrics-server/metrics-certs | ||
| name: metrics-certs | ||
| readOnly: true | ||
|
|
||
| # Add the --metrics-cert-path argument for the metrics server | ||
| - op: add | ||
| path: /spec/template/spec/containers/0/args/0 | ||
| value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs | ||
|
|
||
| # Add the metrics-server certs volume configuration | ||
| - op: add | ||
| path: /spec/template/spec/volumes/- | ||
| value: | ||
| name: metrics-certs | ||
| secret: | ||
| secretName: metrics-server-cert | ||
| optional: false | ||
| items: | ||
| - key: ca.crt | ||
| path: ca.crt | ||
| - key: tls.crt | ||
| path: tls.crt | ||
| - key: tls.key | ||
| path: tls.key |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -13,4 +13,4 @@ spec: | |
| protocol: TCP | ||
| targetPort: 8443 | ||
| selector: | ||
| control-plane: controller-manager | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters