From 0113cf25f53d19b0d05a6ebbe7171a5d44b65f23 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 24 Dec 2024 00:35:19 +0800 Subject: [PATCH 1/8] feat: setup admission webhook --- cmd/operator/app/command.go | 25 ++++++++++++++++++ cmd/operator/app/options/options.go | 39 ++++++++++++++++++----------- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/cmd/operator/app/command.go b/cmd/operator/app/command.go index a381109..0ffae95 100644 --- a/cmd/operator/app/command.go +++ b/cmd/operator/app/command.go @@ -20,6 +20,7 @@ import ( monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/spf13/cobra" + admissionv1beta1 "k8s.io/api/admission/v1beta1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -30,6 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/GreptimeTeam/greptimedb-operator/apis/v1alpha1" "github.com/GreptimeTeam/greptimedb-operator/cmd/operator/app/options" @@ -63,6 +65,9 @@ func init() { // Add [PodMetrics](https://github.com/kubernetes/metrics/blob/master/pkg/apis/metrics/v1beta1/types.go) for fetching PodMetrics from metrics-server. utilruntime.Must(podmetricsv1beta1.AddToScheme(scheme)) + // Add admission webhook scheme. + utilruntime.Must(admissionv1beta1.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme } @@ -77,6 +82,14 @@ func NewOperatorCommand() *cobra.Command { setupLog := ctrl.Log.WithName("setup") cfg := ctrl.GetConfigOrDie() + webhookServer := webhook.NewServer(webhook.Options{}) + if o.EnableAdmissionWebhook { + webhookServerOptions := webhook.Options{ + Port: o.AdmissionWebhookPort, + CertDir: o.AdmissionWebhookCertDir, + } + webhookServer = webhook.NewServer(webhookServerOptions) + } mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, HealthProbeBindAddress: o.HealthProbeAddr, @@ -85,6 +98,7 @@ func NewOperatorCommand() *cobra.Command { Metrics: metricsserver.Options{ BindAddress: o.MetricsAddr, }, + WebhookServer: webhookServer, }) if err != nil { setupLog.Error(err, "unable to start manager") @@ -111,6 +125,17 @@ func NewOperatorCommand() *cobra.Command { os.Exit(1) } + if o.EnableAdmissionWebhook { + if err := (&v1alpha1.GreptimeDBCluster{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to setup admission webhook", "controller", "greptimedbcluster") + os.Exit(1) + } + if err := (&v1alpha1.GreptimeDBStandalone{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to setup admission webhook", "controller", "greptimedbstandalone") + os.Exit(1) + } + } + if o.EnableAPIServer { server, err := apiserver.NewServer(mgr, &apiserver.Options{ Port: o.APIServerPort, diff --git a/cmd/operator/app/options/options.go b/cmd/operator/app/options/options.go index a4e155c..306dbe5 100644 --- a/cmd/operator/app/options/options.go +++ b/cmd/operator/app/options/options.go @@ -19,27 +19,35 @@ import ( ) const ( - defaultMetricsAddr = ":8080" - defaultHealthProbeAddr = ":9494" - defaultAPIServerPort = 8081 + defaultMetricsAddr = ":8080" + defaultHealthProbeAddr = ":9494" + defaultAPIServerPort = 8081 + defaultAdmissionWebhookPort = 8082 + defaultAdmissionWebhookCertDir = "/etc/webhook-server-cert" ) type Options struct { - MetricsAddr string - HealthProbeAddr string - EnableLeaderElection bool - EnableAPIServer bool - APIServerPort int32 - EnablePodMetrics bool + MetricsAddr string + HealthProbeAddr string + EnableLeaderElection bool + EnableAPIServer bool + APIServerPort int32 + EnablePodMetrics bool + EnableAdmissionWebhook bool + AdmissionWebhookPort int + AdmissionWebhookCertDir string } func NewDefaultOptions() *Options { return &Options{ - MetricsAddr: defaultMetricsAddr, - HealthProbeAddr: defaultHealthProbeAddr, - APIServerPort: defaultAPIServerPort, - EnableAPIServer: false, - EnablePodMetrics: false, + MetricsAddr: defaultMetricsAddr, + HealthProbeAddr: defaultHealthProbeAddr, + APIServerPort: defaultAPIServerPort, + EnableAPIServer: false, + EnablePodMetrics: false, + EnableAdmissionWebhook: false, + AdmissionWebhookPort: defaultAdmissionWebhookPort, + AdmissionWebhookCertDir: defaultAdmissionWebhookCertDir, } } @@ -50,4 +58,7 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&o.EnableAPIServer, "enable-apiserver", o.EnableAPIServer, "Enable API server for GreptimeDB operator.") fs.Int32Var(&o.APIServerPort, "apiserver-port", o.APIServerPort, "The port the API server binds to.") fs.BoolVar(&o.EnablePodMetrics, "enable-pod-metrics", o.EnablePodMetrics, "Enable fetching PodMetrics from metrics-server.") + fs.BoolVar(&o.EnableAdmissionWebhook, "enable-admission-webhook", o.EnableAdmissionWebhook, "Enable admission webhook for GreptimeDB operator.") + fs.IntVar(&o.AdmissionWebhookPort, "admission-webhook-port", o.AdmissionWebhookPort, "The port the admission webhook binds to.") + fs.StringVar(&o.AdmissionWebhookCertDir, "admission-webhook-cert-dir", o.AdmissionWebhookCertDir, "The directory that contains the server key and certificate.") } From 6a1abedb7cd0ef1b718c9e42a8b6f311aa3b8a0c Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 24 Dec 2024 00:52:58 +0800 Subject: [PATCH 2/8] chore: change package version --- cmd/operator/app/command.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/operator/app/command.go b/cmd/operator/app/command.go index 0ffae95..e883e09 100644 --- a/cmd/operator/app/command.go +++ b/cmd/operator/app/command.go @@ -20,7 +20,7 @@ import ( monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/spf13/cobra" - admissionv1beta1 "k8s.io/api/admission/v1beta1" + admissionv1 "k8s.io/api/admission/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -66,7 +66,7 @@ func init() { utilruntime.Must(podmetricsv1beta1.AddToScheme(scheme)) // Add admission webhook scheme. - utilruntime.Must(admissionv1beta1.AddToScheme(scheme)) + utilruntime.Must(admissionv1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } From 4f3f0331786a09f48171f0fe5b4a6a3a402f1b1a Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 24 Dec 2024 15:15:53 +0800 Subject: [PATCH 3/8] fix: e2e failed --- config/manager/manager.yaml | 1 + manifests/bundle.yaml | 1 + tests/e2e/setup/create-cluster.sh | 10 ++++++++++ 3 files changed, 12 insertions(+) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 53accf7..239db3e 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -44,6 +44,7 @@ spec: args: - --enable-leader-election image: controller:latest + imagePullPolicy: IfNotPresent name: manager livenessProbe: httpGet: diff --git a/manifests/bundle.yaml b/manifests/bundle.yaml index d5cbaa8..96154c5 100644 --- a/manifests/bundle.yaml +++ b/manifests/bundle.yaml @@ -22475,6 +22475,7 @@ spec: command: - greptimedb-operator image: greptime/greptimedb-operator:latest + imagePullPolicy: IfNotPresent livenessProbe: httpGet: path: /healthz diff --git a/tests/e2e/setup/create-cluster.sh b/tests/e2e/setup/create-cluster.sh index b14905f..b4b9321 100755 --- a/tests/e2e/setup/create-cluster.sh +++ b/tests/e2e/setup/create-cluster.sh @@ -273,6 +273,16 @@ function wait_all_service_ready() { # Wait for kafka to be ready. check_kafka_cluster_status + kubectl get pods -A | grep -E 'e2e|greptimedb-operator' | awk '{print $2 " " $1}' | while read -r line; do + namespace=$(echo "$line" | awk '{print $2}') + pod=$(echo "$line" | awk '{print $1}') + echo "===> Describing pod $pod in namespace $namespace" + kubectl describe pod "$pod" -n "$namespace" + echo "===> Start dumping logs for pod $pod in namespace $namespace" + kubectl logs "$pod" -n "$namespace" + echo "<=== Finish dumping logs for pod $pod in namespace $namespace" + done + # Wait for greptimedb-operator to be ready. kubectl rollout \ status deployment/greptimedb-operator \ From d5f9390926d7165aeac8909cd21c86ba9b4fe187 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 24 Dec 2024 15:28:16 +0800 Subject: [PATCH 4/8] fix: e2e failed --- tests/e2e/setup/create-cluster.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/e2e/setup/create-cluster.sh b/tests/e2e/setup/create-cluster.sh index b4b9321..7c86e4c 100755 --- a/tests/e2e/setup/create-cluster.sh +++ b/tests/e2e/setup/create-cluster.sh @@ -263,15 +263,19 @@ function wait_for() { function wait_all_service_ready() { echo -e "${GREEN}=> Wait for all services to be ready...${RESET}" + echo -e "${GREEN}<= Wait for etcd services to be ready.${RESET}" # Wait for etcd to be ready. kubectl wait \ --for=condition=Ready \ pod -l app.kubernetes.io/instance=etcd \ -n "$ETCD_NAMESPACE" \ --timeout="$DEFAULT_TIMEOUT" + echo -e "${GREEN}<= The etcd services is ready.${RESET}" + echo -e "${GREEN}<= Wait for kafka services to be ready.${RESET}" # Wait for kafka to be ready. check_kafka_cluster_status + echo -e "${GREEN}<= The kafka services is ready.${RESET}" kubectl get pods -A | grep -E 'e2e|greptimedb-operator' | awk '{print $2 " " $1}' | while read -r line; do namespace=$(echo "$line" | awk '{print $2}') @@ -283,11 +287,13 @@ function wait_all_service_ready() { echo "<=== Finish dumping logs for pod $pod in namespace $namespace" done + echo -e "${GREEN}<= Wait for greptimedb-operator services to be ready.${RESET}" # Wait for greptimedb-operator to be ready. kubectl rollout \ status deployment/greptimedb-operator \ -n greptimedb-admin \ --timeout="$DEFAULT_TIMEOUT" + echo -e "${GREEN}<= The greptimedb-operator services is ready.${RESET}" echo -e "${GREEN}<= All services are ready.${RESET}" } From 949783a9d227738feea091fcc09bc6f3862be79b Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 24 Dec 2024 15:49:50 +0800 Subject: [PATCH 5/8] fix: e2e failed --- tests/e2e/setup/create-cluster.sh | 16 ---------------- tests/e2e/setup/kafka-wal.yaml | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/tests/e2e/setup/create-cluster.sh b/tests/e2e/setup/create-cluster.sh index 7c86e4c..b14905f 100755 --- a/tests/e2e/setup/create-cluster.sh +++ b/tests/e2e/setup/create-cluster.sh @@ -263,37 +263,21 @@ function wait_for() { function wait_all_service_ready() { echo -e "${GREEN}=> Wait for all services to be ready...${RESET}" - echo -e "${GREEN}<= Wait for etcd services to be ready.${RESET}" # Wait for etcd to be ready. kubectl wait \ --for=condition=Ready \ pod -l app.kubernetes.io/instance=etcd \ -n "$ETCD_NAMESPACE" \ --timeout="$DEFAULT_TIMEOUT" - echo -e "${GREEN}<= The etcd services is ready.${RESET}" - echo -e "${GREEN}<= Wait for kafka services to be ready.${RESET}" # Wait for kafka to be ready. check_kafka_cluster_status - echo -e "${GREEN}<= The kafka services is ready.${RESET}" - - kubectl get pods -A | grep -E 'e2e|greptimedb-operator' | awk '{print $2 " " $1}' | while read -r line; do - namespace=$(echo "$line" | awk '{print $2}') - pod=$(echo "$line" | awk '{print $1}') - echo "===> Describing pod $pod in namespace $namespace" - kubectl describe pod "$pod" -n "$namespace" - echo "===> Start dumping logs for pod $pod in namespace $namespace" - kubectl logs "$pod" -n "$namespace" - echo "<=== Finish dumping logs for pod $pod in namespace $namespace" - done - echo -e "${GREEN}<= Wait for greptimedb-operator services to be ready.${RESET}" # Wait for greptimedb-operator to be ready. kubectl rollout \ status deployment/greptimedb-operator \ -n greptimedb-admin \ --timeout="$DEFAULT_TIMEOUT" - echo -e "${GREEN}<= The greptimedb-operator services is ready.${RESET}" echo -e "${GREEN}<= All services are ready.${RESET}" } diff --git a/tests/e2e/setup/kafka-wal.yaml b/tests/e2e/setup/kafka-wal.yaml index 2c8c2a3..60e6085 100644 --- a/tests/e2e/setup/kafka-wal.yaml +++ b/tests/e2e/setup/kafka-wal.yaml @@ -26,7 +26,7 @@ metadata: strimzi.io/kraft: enabled spec: kafka: - version: 3.7.0 + version: 3.9.0 metadataVersion: 3.7-IV4 listeners: - name: plain From c5d53b39c05f04b9e5a7e5e73361a3762e87c598 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 24 Dec 2024 15:53:07 +0800 Subject: [PATCH 6/8] chore: change admission webhook cert directory --- cmd/operator/app/options/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/operator/app/options/options.go b/cmd/operator/app/options/options.go index 306dbe5..09c6fa1 100644 --- a/cmd/operator/app/options/options.go +++ b/cmd/operator/app/options/options.go @@ -23,7 +23,7 @@ const ( defaultHealthProbeAddr = ":9494" defaultAPIServerPort = 8081 defaultAdmissionWebhookPort = 8082 - defaultAdmissionWebhookCertDir = "/etc/webhook-server-cert" + defaultAdmissionWebhookCertDir = "/etc/greptimedb/admission-webhook-tls" ) type Options struct { From bef5aa14d643f52dd65d2aaf3952847e90de8b32 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 24 Dec 2024 16:27:32 +0800 Subject: [PATCH 7/8] chore: change kafka metadataVersion to 3.9 --- tests/e2e/setup/kafka-wal.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/setup/kafka-wal.yaml b/tests/e2e/setup/kafka-wal.yaml index 60e6085..135df66 100644 --- a/tests/e2e/setup/kafka-wal.yaml +++ b/tests/e2e/setup/kafka-wal.yaml @@ -27,7 +27,7 @@ metadata: spec: kafka: version: 3.9.0 - metadataVersion: 3.7-IV4 + metadataVersion: 3.9 listeners: - name: plain port: 9092 From a71adc28f3f6e429daf64cb7ac992fe6303b59c8 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 24 Dec 2024 16:34:49 +0800 Subject: [PATCH 8/8] fix: change kafka metadataVersion version to string --- tests/e2e/setup/kafka-wal.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/setup/kafka-wal.yaml b/tests/e2e/setup/kafka-wal.yaml index 135df66..04e2318 100644 --- a/tests/e2e/setup/kafka-wal.yaml +++ b/tests/e2e/setup/kafka-wal.yaml @@ -27,7 +27,7 @@ metadata: spec: kafka: version: 3.9.0 - metadataVersion: 3.9 + metadataVersion: "3.9" listeners: - name: plain port: 9092