From 7369a80fc555392fa2cdce9eb39045ef532f1d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=92=E6=83=85=E7=86=8A?= <2669184984@qq.com> Date: Mon, 2 Dec 2024 18:19:08 +0800 Subject: [PATCH 1/2] feat:devbox controller support websocket feat:devbox controller support websocket --- .../devbox/api/v1alpha1/devbox_types.go | 8 +- .../internal/controller/devbox_controller.go | 268 +++++++++++++++--- .../internal/controller/helper/devbox.go | 44 ++- 3 files changed, 274 insertions(+), 46 deletions(-) diff --git a/controllers/devbox/api/v1alpha1/devbox_types.go b/controllers/devbox/api/v1alpha1/devbox_types.go index 5c82533e142..27d3907efcb 100644 --- a/controllers/devbox/api/v1alpha1/devbox_types.go +++ b/controllers/devbox/api/v1alpha1/devbox_types.go @@ -48,8 +48,9 @@ const ( type NetworkType string const ( - NetworkTypeNodePort NetworkType = "NodePort" - NetworkTypeTailnet NetworkType = "Tailnet" + NetworkTypeNodePort NetworkType = "NodePort" + NetworkTypeTailnet NetworkType = "Tailnet" + NetworkTypeWebSocket NetworkType = "WebSocket" ) type ResourceList map[ResourceName]resource.Quantity @@ -123,6 +124,9 @@ type NetworkStatus struct { // +kubebuilder:validation:Optional NodePort int32 `json:"nodePort"` + // +kubebuilder:validation:Optional + WebSocket string `json:"webSocket"` + // todo TailNet // +kubebuilder:validation:Optional TailNet string `json:"tailnet"` diff --git a/controllers/devbox/internal/controller/devbox_controller.go b/controllers/devbox/internal/controller/devbox_controller.go index 76fcb82d9da..325bc643dfe 100644 --- a/controllers/devbox/internal/controller/devbox_controller.go +++ b/controllers/devbox/internal/controller/devbox_controller.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + networkingv1 "k8s.io/api/networking/v1" "time" devboxv1alpha1 "github.com/labring/sealos/controllers/devbox/api/v1alpha1" @@ -49,8 +50,8 @@ type DevboxReconciler struct { RequestMemoryRate float64 RequestEphemeralStorage string LimitEphemeralStorage string - - DebugMode bool + WebSocketImage string + DebugMode bool client.Client Scheme *runtime.Scheme @@ -125,20 +126,13 @@ func (r *DevboxReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr logger.Info("sync secret success") r.Recorder.Eventf(devbox, corev1.EventTypeNormal, "Sync secret success", "Sync secret success") - // create service if network type is NodePort - if devbox.Spec.NetworkSpec.Type == devboxv1alpha1.NetworkTypeNodePort { - logger.Info("syncing service") - if err := r.Get(ctx, req.NamespacedName, devbox); err != nil { - return ctrl.Result{}, err - } - if err := r.syncService(ctx, devbox, recLabels); err != nil { - logger.Error(err, "sync service failed") - r.Recorder.Eventf(devbox, corev1.EventTypeWarning, "Sync service failed", "%v", err) - return ctrl.Result{}, err - } - logger.Info("sync service success") - r.Recorder.Eventf(devbox, corev1.EventTypeNormal, "Sync service success", "Sync service success") + logger.Info("syncing network") + if err := r.syncNetwork(ctx, devbox, recLabels); err != nil { + logger.Error(err, "sync network failed") + r.Recorder.Eventf(devbox, corev1.EventTypeWarning, "Sync network failed", "%v", err) + return ctrl.Result{}, err } + logger.Info("sync network success") // create or update pod logger.Info("syncing pod") @@ -342,31 +336,8 @@ func (r *DevboxReconciler) syncPod(ctx context.Context, devbox *devboxv1alpha1.D return nil } -func (r *DevboxReconciler) syncService(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string) error { - runtimecr, err := r.getRuntime(ctx, devbox) - if err != nil { - return err - } - var servicePorts []corev1.ServicePort - for _, port := range runtimecr.Spec.Config.Ports { - servicePorts = append(servicePorts, corev1.ServicePort{ - Name: port.Name, - Port: port.ContainerPort, - TargetPort: intstr.FromInt32(port.ContainerPort), - Protocol: port.Protocol, - }) - } - if len(servicePorts) == 0 { - //use the default value - servicePorts = []corev1.ServicePort{ - { - Name: "devbox-ssh-port", - Port: 22, - TargetPort: intstr.FromInt32(22), - Protocol: corev1.ProtocolTCP, - }, - } - } +func (r *DevboxReconciler) syncNodePortNetwork(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { + var err error expectServiceSpec := corev1.ServiceSpec{ Selector: recLabels, Type: corev1.ServiceTypeNodePort, @@ -439,6 +410,223 @@ func (r *DevboxReconciler) getRuntime(ctx context.Context, devbox *devboxv1alpha return runtimecr, nil } +func (r *DevboxReconciler) syncNetwork(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string) error { + runtimecr, err := r.getRuntime(ctx, devbox) + if err != nil { + return err + } + var servicePorts []corev1.ServicePort + for _, port := range runtimecr.Spec.Config.Ports { + servicePorts = append(servicePorts, corev1.ServicePort{ + Name: port.Name, + Port: port.ContainerPort, + TargetPort: intstr.FromInt32(port.ContainerPort), + Protocol: port.Protocol, + }) + } + if len(servicePorts) == 0 { + //use the default value + servicePorts = []corev1.ServicePort{ + { + Name: "devbox-ssh-port", + Port: 22, + TargetPort: intstr.FromInt32(22), + Protocol: corev1.ProtocolTCP, + }, + } + } + switch devbox.Spec.NetworkSpec.Type { + case devboxv1alpha1.NetworkTypeNodePort: + return r.syncNodePortNetwork(ctx, devbox, recLabels, servicePorts) + case devboxv1alpha1.NetworkTypeWebSocket: + return r.syncWebSocketNetwork(ctx, devbox, recLabels, servicePorts) + } + return nil +} + +func (r *DevboxReconciler) syncWebSocketNetwork(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { + if err := r.syncPodSvc(ctx, devbox, recLabels, servicePorts); err != nil { + return err + } + if err := r.syncProxyPod(ctx, devbox, recLabels, servicePorts); err != nil { + return err + } + if err := r.syncProxySvc(ctx, devbox, recLabels, servicePorts); err != nil { + return err + } + if err := r.syncProxyIngress(ctx, devbox); err != nil { + return err + } + devbox.Status.Network.Type = devboxv1alpha1.NetworkTypeWebSocket + devbox.Status.Network.WebSocket = devbox.Name + "-proxy-ingress" + return r.Status().Update(ctx, devbox) +} + +func (r *DevboxReconciler) syncProxyIngress(ctx context.Context, devbox *devboxv1alpha1.Devbox) error { + wsIngress := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: devbox.Name + "-proxy-ingress", + Namespace: devbox.Namespace, + }, + Spec: networkingv1.IngressSpec{ + Rules: []networkingv1.IngressRule{ + { + Host: devbox.Name + ".sealoshzh.site", + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: new(networkingv1.PathType), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: devbox.Name + "-proxy-svc", + Port: networkingv1.ServiceBackendPort{ + Number: 22, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, wsIngress, func() error { + return controllerutil.SetControllerReference(devbox, wsIngress, r.Scheme) + }); err != nil { + return err + } + return nil +} + +func (r *DevboxReconciler) syncProxySvc(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { + runtimecr, err := r.getRuntime(ctx, devbox) + if err != nil { + return err + } + + expectServiceSpec := corev1.ServiceSpec{ + Selector: helper.GenerateProxyPodLabels(devbox, runtimecr), + Type: corev1.ServiceTypeClusterIP, + Ports: servicePorts, + } + proxySvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: devbox.Name + "-proxy-svc", + Namespace: devbox.Namespace, + Labels: helper.GenerateProxyPodLabels(devbox, runtimecr), + }, + } + + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, proxySvc, func() error { + // only update some specific fields + proxySvc.Spec.Selector = expectServiceSpec.Selector + proxySvc.Spec.Type = expectServiceSpec.Type + if len(proxySvc.Spec.Ports) == 0 { + proxySvc.Spec.Ports = expectServiceSpec.Ports + } else { + proxySvc.Spec.Ports[0].Name = expectServiceSpec.Ports[0].Name + proxySvc.Spec.Ports[0].Port = expectServiceSpec.Ports[0].Port + proxySvc.Spec.Ports[0].TargetPort = expectServiceSpec.Ports[0].TargetPort + proxySvc.Spec.Ports[0].Protocol = expectServiceSpec.Ports[0].Protocol + } + return controllerutil.SetControllerReference(devbox, proxySvc, r.Scheme) + }); err != nil { + return err + } + return nil +} + +func (r *DevboxReconciler) syncProxyPod(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { + runtimecr, err := r.getRuntime(ctx, devbox) + if err != nil { + return err + } + + sshPort := "22" + for _, port := range servicePorts { + if port.Name == "devbox-ssh-port" { + sshPort = port.TargetPort.String() + break + } + } + + wsPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: devbox.Name + "-proxy-pod", + Namespace: devbox.Namespace, + Labels: helper.GenerateProxyPodLabels(devbox, runtimecr), + Annotations: helper.GeneratePodAnnotations(devbox, runtimecr), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "ws-proxy", + Image: r.WebSocketImage, + Args: []string{ + "server", + "--port=", + fmt.Sprintf("--port=%s", sshPort), + fmt.Sprintf("--proxy=%s", devbox.Name+"-pod-svc:22"), + "-v=true", + }, + Resources: helper.GenerateProxyPodResourceRequirements(), + }, + }, + }, + } + // if devbox is running, create the pod + if devbox.Spec.State == devboxv1alpha1.DevboxStateRunning { + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, wsPod, func() error { + return controllerutil.SetControllerReference(devbox, wsPod, r.Scheme) + }); err != nil { + return err + } + } + return nil +} + +func (r *DevboxReconciler) syncPodSvc(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { + runtimecr, err := r.getRuntime(ctx, devbox) + if err != nil { + return err + } + + expectServiceSpec := corev1.ServiceSpec{ + Selector: recLabels, + Type: corev1.ServiceTypeClusterIP, + Ports: servicePorts, + } + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: devbox.Name + "-pod-svc", + Namespace: devbox.Namespace, + Labels: helper.GenerateProxyPodLabels(devbox, runtimecr), + }, + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, func() error { + // only update some specific fields + service.Spec.Selector = expectServiceSpec.Selector + service.Spec.Type = expectServiceSpec.Type + if len(service.Spec.Ports) == 0 { + service.Spec.Ports = expectServiceSpec.Ports + } else { + service.Spec.Ports[0].Name = expectServiceSpec.Ports[0].Name + service.Spec.Ports[0].Port = expectServiceSpec.Ports[0].Port + service.Spec.Ports[0].TargetPort = expectServiceSpec.Ports[0].TargetPort + service.Spec.Ports[0].Protocol = expectServiceSpec.Ports[0].Protocol + } + return controllerutil.SetControllerReference(devbox, service, r.Scheme) + }); err != nil { + return err + } + return nil + +} + // create a new pod, add predicated status to nextCommitHistory func (r *DevboxReconciler) createPod(ctx context.Context, devbox *devboxv1alpha1.Devbox, expectPod *corev1.Pod, nextCommitHistory *devboxv1alpha1.CommitHistory) error { nextCommitHistory.Status = devboxv1alpha1.CommitStatusPending diff --git a/controllers/devbox/internal/controller/helper/devbox.go b/controllers/devbox/internal/controller/helper/devbox.go index bd00faee4c8..b32b4d10f70 100644 --- a/controllers/devbox/internal/controller/helper/devbox.go +++ b/controllers/devbox/internal/controller/helper/devbox.go @@ -36,6 +36,7 @@ import ( const ( DevBoxPartOf = "devbox" + ProxyPod = "proxy" ) func GeneratePodLabels(devbox *devboxv1alpha1.Devbox, runtime *devboxv1alpha1.Runtime) map[string]string { @@ -62,6 +63,31 @@ func GeneratePodLabels(devbox *devboxv1alpha1.Devbox, runtime *devboxv1alpha1.Ru return labels } +func GenerateProxyPodLabels(devbox *devboxv1alpha1.Devbox, runtime *devboxv1alpha1.Runtime) map[string]string { + labels := make(map[string]string) + + if runtime.Spec.Config.Labels != nil { + for k, v := range runtime.Spec.Config.Labels { + labels[k] = v + } + } + if devbox.Spec.ExtraLabels != nil { + for k, v := range devbox.Spec.ExtraLabels { + labels[k] = v + } + } + recLabels := label.RecommendedLabels(&label.Recommended{ + Name: devbox.Name, + ManagedBy: label.DefaultManagedBy, + PartOf: DevBoxPartOf, + Component: ProxyPod, + }) + for k, v := range recLabels { + labels[k] = v + } + return labels +} + func GeneratePodAnnotations(devbox *devboxv1alpha1.Devbox, runtime *devboxv1alpha1.Runtime) map[string]string { annotations := make(map[string]string) if runtime.Spec.Config.Annotations != nil { @@ -398,10 +424,7 @@ func GenerateSSHVolume(devbox *devboxv1alpha1.Devbox) corev1.Volume { } } -func GenerateResourceRequirements(devbox *devboxv1alpha1.Devbox, - requestCPURate, requestMemoryRate float64, - requestEphemeralStorage, limitEphemeralStorage string, -) corev1.ResourceRequirements { +func GenerateResourceRequirements(devbox *devboxv1alpha1.Devbox, requestCPURate, requestMemoryRate float64, requestEphemeralStorage, limitEphemeralStorage string) corev1.ResourceRequirements { return corev1.ResourceRequirements{ Requests: calculateResourceRequest( corev1.ResourceList{ @@ -419,6 +442,19 @@ func GenerateResourceRequirements(devbox *devboxv1alpha1.Devbox, } } +func GenerateProxyPodResourceRequirements() corev1.ResourceRequirements { + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + } +} + func IsExceededQuotaError(err error) bool { return strings.Contains(err.Error(), "exceeded quota") } From 676a49593f4b747a7d1ebd46c08b1f5186e9a5d9 Mon Sep 17 00:00:00 2001 From: bearslyricattack Date: Mon, 2 Dec 2024 10:19:43 +0000 Subject: [PATCH 2/2] chore(fmt): format code Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> feat:devbox controller support websocket docs: change blog url (#5251) Signed-off-by: Carson Yang feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket chore(fmt): format code Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket feat:devbox controller support websocket chore(fmt): format code Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> feat:websocket auto shutdown feat:websocket auto shutdown chore(fmt): format code Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown chore(fmt): format code Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown feat:websocket auto shutdown Delete tencent9825051126872297210.txt --- .../devbox/api/v1alpha1/devbox_types.go | 15 +- controllers/devbox/cmd/main.go | 18 ++ .../internal/controller/devbox_controller.go | 300 +++++++++++++----- .../internal/controller/helper/devbox.go | 10 +- 4 files changed, 262 insertions(+), 81 deletions(-) diff --git a/controllers/devbox/api/v1alpha1/devbox_types.go b/controllers/devbox/api/v1alpha1/devbox_types.go index 27d3907efcb..d9a30d4df33 100644 --- a/controllers/devbox/api/v1alpha1/devbox_types.go +++ b/controllers/devbox/api/v1alpha1/devbox_types.go @@ -64,12 +64,20 @@ type RuntimeRef struct { type NetworkSpec struct { // +kubebuilder:validation:Required - // +kubebuilder:validation:Enum=NodePort;Tailnet + // +kubebuilder:validation:Enum=NodePort;Tailnet;WebSocket Type NetworkType `json:"type"` // +kubebuilder:validation:Optional ExtraPorts []corev1.ContainerPort `json:"extraPorts"` } +type AutoShutdownSpec struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + Enable bool `json:"type"` + // +kubebuilder:validation:Optional + Time string `json:"time"` +} + // DevboxSpec defines the desired state of Devbox type DevboxSpec struct { // +kubebuilder:validation:Required @@ -88,6 +96,9 @@ type DevboxSpec struct { // +kubebuilder:validation:Required NetworkSpec NetworkSpec `json:"network,omitempty"` + // +kubebuilder:validation:Optional + AutoShutdownSpec AutoShutdownSpec `json:"autoShutdown,omitempty"` + // todo add rewrite labels and annotations... // +kubebuilder:validation:Optional ExtraLabels map[string]string `json:"extraLabels,omitempty"` @@ -118,7 +129,7 @@ type DevboxSpec struct { type NetworkStatus struct { // +kubebuilder:default=NodePort - // +kubebuilder:validation:Enum=NodePort;Tailnet + // +kubebuilder:validation:Enum=NodePort;Tailnet;WebSocket Type NetworkType `json:"type"` // +kubebuilder:validation:Optional diff --git a/controllers/devbox/cmd/main.go b/controllers/devbox/cmd/main.go index 62416e9441d..0931cdd8191 100644 --- a/controllers/devbox/cmd/main.go +++ b/controllers/devbox/cmd/main.go @@ -74,6 +74,12 @@ func main() { var requestEphemeralStorage string var limitEphemeralStorage string var debugMode bool + var webSocketImage string + var websocketProxyDomain string + var ingressClass string + var enableAutoShutdown bool + var shutdownServerKey string + var shutdownServerAddr string flag.StringVar(®istryAddr, "registry-addr", "sealos.hub:5000", "The address of the registry") flag.StringVar(®istryUser, "registry-user", "admin", "The user of the registry") flag.StringVar(®istryPassword, "registry-password", "passw0rd", "The password of the registry") @@ -93,6 +99,12 @@ func main() { flag.Float64Var(&requestMemoryRate, "request-memory-rate", 10, "The request rate of memory limit in devbox.") flag.StringVar(&requestEphemeralStorage, "request-ephemeral-storage", "500Mi", "The request value of ephemeral storage in devbox.") flag.StringVar(&limitEphemeralStorage, "limit-ephemeral-storage", "10Gi", "The limit value of ephemeral storage in devbox.") + flag.StringVar(&webSocketImage, "websocket-image", "cbluebird/wst:v0.0.4", "The image name of devbox websocket proxy pod.") + flag.StringVar(&websocketProxyDomain, "websocket-proxy-domain", "sealoshzh.site", "The websocket proxy domain of devbox ingress.") + flag.StringVar(&ingressClass, "ingress-class", "nginx", "The ingress class name.") + flag.BoolVar(&enableAutoShutdown, "enable-auto-shutdown", true, "If set, Devbox auto shutdown will be enabled.") + flag.StringVar(&shutdownServerKey, "shutdown-server-key", "sealos-devbox-shutdown", "The server key used to shutdown the server.") + flag.StringVar(&shutdownServerAddr, "shutdown-server-addr", "http://shutdown-service:8082", "The shutdown server address.") opts := zap.Options{ Development: true, } @@ -192,6 +204,12 @@ func main() { RequestEphemeralStorage: requestEphemeralStorage, LimitEphemeralStorage: limitEphemeralStorage, DebugMode: debugMode, + WebSocketImage: webSocketImage, + WebsocketProxyDomain: websocketProxyDomain, + IngressClass: ingressClass, + EnableAutoShutdown: enableAutoShutdown, + ShutdownServerKey: shutdownServerKey, + ShutdownServerAddr: shutdownServerAddr, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Devbox") os.Exit(1) diff --git a/controllers/devbox/internal/controller/devbox_controller.go b/controllers/devbox/internal/controller/devbox_controller.go index 325bc643dfe..b057ac6f2b4 100644 --- a/controllers/devbox/internal/controller/devbox_controller.go +++ b/controllers/devbox/internal/controller/devbox_controller.go @@ -19,13 +19,17 @@ package controller import ( "context" "fmt" - networkingv1 "k8s.io/api/networking/v1" + "strconv" "time" - devboxv1alpha1 "github.com/labring/sealos/controllers/devbox/api/v1alpha1" - "github.com/labring/sealos/controllers/devbox/internal/controller/helper" - "github.com/labring/sealos/controllers/devbox/label" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + networkingv1 "k8s.io/api/networking/v1" + + "github.com/golang-jwt/jwt/v5" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -35,6 +39,10 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" + devboxv1alpha1 "github.com/labring/sealos/controllers/devbox/api/v1alpha1" + "github.com/labring/sealos/controllers/devbox/internal/controller/helper" + "github.com/labring/sealos/controllers/devbox/label" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -52,7 +60,11 @@ type DevboxReconciler struct { LimitEphemeralStorage string WebSocketImage string DebugMode bool - + WebsocketProxyDomain string + IngressClass string + EnableAutoShutdown bool + ShutdownServerKey string + ShutdownServerAddr string client.Client Scheme *runtime.Scheme Recorder record.EventRecorder @@ -410,10 +422,10 @@ func (r *DevboxReconciler) getRuntime(ctx context.Context, devbox *devboxv1alpha return runtimecr, nil } -func (r *DevboxReconciler) syncNetwork(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string) error { +func (r *DevboxReconciler) getServicePort(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string) ([]corev1.ServicePort, error) { runtimecr, err := r.getRuntime(ctx, devbox) if err != nil { - return err + return nil, err } var servicePorts []corev1.ServicePort for _, port := range runtimecr.Spec.Config.Ports { @@ -425,7 +437,6 @@ func (r *DevboxReconciler) syncNetwork(ctx context.Context, devbox *devboxv1alph }) } if len(servicePorts) == 0 { - //use the default value servicePorts = []corev1.ServicePort{ { Name: "devbox-ssh-port", @@ -435,6 +446,14 @@ func (r *DevboxReconciler) syncNetwork(ctx context.Context, devbox *devboxv1alph }, } } + return servicePorts, nil +} + +func (r *DevboxReconciler) syncNetwork(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string) error { + servicePorts, err := r.getServicePort(ctx, devbox, recLabels) + if err != nil { + return err + } switch devbox.Spec.NetworkSpec.Type { case devboxv1alpha1.NetworkTypeNodePort: return r.syncNodePortNetwork(ctx, devbox, recLabels, servicePorts) @@ -445,6 +464,10 @@ func (r *DevboxReconciler) syncNetwork(ctx context.Context, devbox *devboxv1alph } func (r *DevboxReconciler) syncWebSocketNetwork(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { + devbox.Status.Network.Type = devboxv1alpha1.NetworkTypeWebSocket + if err := r.Status().Update(ctx, devbox); err != nil { + return err + } if err := r.syncPodSvc(ctx, devbox, recLabels, servicePorts); err != nil { return err } @@ -454,52 +477,66 @@ func (r *DevboxReconciler) syncWebSocketNetwork(ctx context.Context, devbox *dev if err := r.syncProxySvc(ctx, devbox, recLabels, servicePorts); err != nil { return err } - if err := r.syncProxyIngress(ctx, devbox); err != nil { + if hostName, err := r.syncProxyIngress(ctx, devbox); err != nil { return err + } else { + devbox.Status.Network.WebSocket = hostName } - devbox.Status.Network.Type = devboxv1alpha1.NetworkTypeWebSocket - devbox.Status.Network.WebSocket = devbox.Name + "-proxy-ingress" return r.Status().Update(ctx, devbox) } -func (r *DevboxReconciler) syncProxyIngress(ctx context.Context, devbox *devboxv1alpha1.Devbox) error { - wsIngress := &networkingv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: devbox.Name + "-proxy-ingress", - Namespace: devbox.Namespace, +func (r *DevboxReconciler) generateProxyIngressHost() string { + return rand.String(12) + "." + r.WebsocketProxyDomain +} + +func (r *DevboxReconciler) syncProxyIngress(ctx context.Context, devbox *devboxv1alpha1.Devbox) (string, error) { + host := r.generateProxyIngressHost() + + pathType := networkingv1.PathTypePrefix + ingressPath := []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: &pathType, + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: devbox.Name + "-proxy-svc", + Port: networkingv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, }, - Spec: networkingv1.IngressSpec{ - Rules: []networkingv1.IngressRule{ - { - Host: devbox.Name + ".sealoshzh.site", - IngressRuleValue: networkingv1.IngressRuleValue{ - HTTP: &networkingv1.HTTPIngressRuleValue{ - Paths: []networkingv1.HTTPIngressPath{ - { - Path: "/", - PathType: new(networkingv1.PathType), - Backend: networkingv1.IngressBackend{ - Service: &networkingv1.IngressServiceBackend{ - Name: devbox.Name + "-proxy-svc", - Port: networkingv1.ServiceBackendPort{ - Number: 22, - }, - }, - }, - }, - }, - }, + } + + ingressSpec := networkingv1.IngressSpec{ + IngressClassName: &r.IngressClass, + Rules: []networkingv1.IngressRule{ + { + Host: host, + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: ingressPath, }, }, }, }, } + + wsIngress := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: devbox.Name + "-proxy-ingress", + Namespace: devbox.Namespace, + }, + Spec: ingressSpec, + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, wsIngress, func() error { return controllerutil.SetControllerReference(devbox, wsIngress, r.Scheme) }); err != nil { - return err + return "", err } - return nil + + return host, nil } func (r *DevboxReconciler) syncProxySvc(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { @@ -507,11 +544,18 @@ func (r *DevboxReconciler) syncProxySvc(ctx context.Context, devbox *devboxv1alp if err != nil { return err } - + servicePort := []corev1.ServicePort{ + { + Name: "devbox-ssh-port", + Port: 80, + TargetPort: intstr.FromInt32(80), + Protocol: corev1.ProtocolTCP, + }, + } expectServiceSpec := corev1.ServiceSpec{ Selector: helper.GenerateProxyPodLabels(devbox, runtimecr), Type: corev1.ServiceTypeClusterIP, - Ports: servicePorts, + Ports: servicePort, } proxySvc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -519,20 +563,15 @@ func (r *DevboxReconciler) syncProxySvc(ctx context.Context, devbox *devboxv1alp Namespace: devbox.Namespace, Labels: helper.GenerateProxyPodLabels(devbox, runtimecr), }, + Spec: expectServiceSpec, } - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, proxySvc, func() error { - // only update some specific fields proxySvc.Spec.Selector = expectServiceSpec.Selector proxySvc.Spec.Type = expectServiceSpec.Type - if len(proxySvc.Spec.Ports) == 0 { - proxySvc.Spec.Ports = expectServiceSpec.Ports - } else { - proxySvc.Spec.Ports[0].Name = expectServiceSpec.Ports[0].Name - proxySvc.Spec.Ports[0].Port = expectServiceSpec.Ports[0].Port - proxySvc.Spec.Ports[0].TargetPort = expectServiceSpec.Ports[0].TargetPort - proxySvc.Spec.Ports[0].Protocol = expectServiceSpec.Ports[0].Protocol - } + proxySvc.Spec.Ports[0].Name = expectServiceSpec.Ports[0].Name + proxySvc.Spec.Ports[0].Port = expectServiceSpec.Ports[0].Port + proxySvc.Spec.Ports[0].TargetPort = expectServiceSpec.Ports[0].TargetPort + proxySvc.Spec.Ports[0].Protocol = expectServiceSpec.Ports[0].Protocol return controllerutil.SetControllerReference(devbox, proxySvc, r.Scheme) }); err != nil { return err @@ -540,12 +579,63 @@ func (r *DevboxReconciler) syncProxySvc(ctx context.Context, devbox *devboxv1alp return nil } -func (r *DevboxReconciler) syncProxyPod(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { - runtimecr, err := r.getRuntime(ctx, devbox) +func (r *DevboxReconciler) generateProxyPodName(devbox *devboxv1alpha1.Devbox) string { + return devbox.Name + "-proxy-pod" + "-" + rand.String(5) +} + +func (r *DevboxReconciler) generateProxyPodDeploymentName(devbox *devboxv1alpha1.Devbox) string { + return devbox.Name + "-proxy-deployment" +} + +type DevboxClaims struct { + DevboxName string `json:"devbox_name"` + NameSpace string `json:"namespace"` + jwt.RegisteredClaims +} + +func (r *DevboxReconciler) generateProxyPodJWT(ctx context.Context, devbox *devboxv1alpha1.Devbox) (string, error) { + claims := DevboxClaims{ + DevboxName: devbox.Name, + NameSpace: devbox.Namespace, + RegisteredClaims: jwt.RegisteredClaims{ + IssuedAt: jwt.NewNumericDate(time.Now()), + ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour * 7 * 24)), + Issuer: "devbox-controller", + }, + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + signedToken, err := token.SignedString([]byte(r.ShutdownServerKey)) if err != nil { - return err + return "", fmt.Errorf("failed to sign token: %w", err) + } + return signedToken, nil +} + +func (r *DevboxReconciler) generateProxyPodEnv(ctx context.Context, devbox *devboxv1alpha1.Devbox, servicePorts []corev1.ServicePort) ([]corev1.EnvVar, error) { + var envVars []corev1.EnvVar + autoShutdownEnabled := devbox.Spec.AutoShutdownSpec.Enable && r.EnableAutoShutdown + envVars = append(envVars, corev1.EnvVar{ + Name: "ENABLE_AUTO_SHUTDOWN", + Value: strconv.FormatBool(autoShutdownEnabled), + }) + + if autoShutdownEnabled { + envVars = append(envVars, corev1.EnvVar{ + Name: "AUTO_SHUTDOWN_INTERVAL", + Value: devbox.Spec.AutoShutdownSpec.Time, + }) } + token, err := r.generateProxyPodJWT(ctx, devbox) + if err != nil { + return nil, err + } + + envVars = append(envVars, corev1.EnvVar{ + Name: "JWT_TOKEN", + Value: token, + }) + sshPort := "22" for _, port := range servicePorts { if port.Name == "devbox-ssh-port" { @@ -553,36 +643,96 @@ func (r *DevboxReconciler) syncProxyPod(ctx context.Context, devbox *devboxv1alp break } } + envVars = append(envVars, corev1.EnvVar{ + Name: "TARGET", + Value: fmt.Sprintf("%s-pod-svc:%s", devbox.Name, sshPort), + }) + + envVars = append(envVars, corev1.EnvVar{ + Name: "LISTEN", + Value: "0.0.0.0:80", + }) - wsPod := &corev1.Pod{ + envVars = append(envVars, corev1.EnvVar{ + Name: "AUTO_SHUTDOWN_SERVICE_URL", + Value: r.ShutdownServerAddr, + }) + + return envVars, nil +} + +func (r *DevboxReconciler) generateProxyPodDeployment(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) (*appsv1.Deployment, error) { + runtimecr, err := r.getRuntime(ctx, devbox) + if err != nil { + return nil, err + } + + podEnv, err := r.generateProxyPodEnv(ctx, devbox, servicePorts) + if err != nil { + return nil, err + } + + podSpec := corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "ws-proxy", + Image: r.WebSocketImage, + Env: podEnv, + Resources: helper.GenerateProxyPodResourceRequirements(), + }, + }, + } + + replicas := int32(1) + return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: devbox.Name + "-proxy-pod", + Name: r.generateProxyPodDeploymentName(devbox), Namespace: devbox.Namespace, Labels: helper.GenerateProxyPodLabels(devbox, runtimecr), Annotations: helper.GeneratePodAnnotations(devbox, runtimecr), }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "ws-proxy", - Image: r.WebSocketImage, - Args: []string{ - "server", - "--port=", - fmt.Sprintf("--port=%s", sshPort), - fmt.Sprintf("--proxy=%s", devbox.Name+"-pod-svc:22"), - "-v=true", - }, - Resources: helper.GenerateProxyPodResourceRequirements(), + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: helper.GenerateProxyPodLabels(devbox, runtimecr), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.generateProxyPodName(devbox), + Namespace: devbox.Namespace, + Labels: helper.GenerateProxyPodLabels(devbox, runtimecr), + Annotations: helper.GeneratePodAnnotations(devbox, runtimecr), }, + Spec: podSpec, }, }, - } - // if devbox is running, create the pod + }, nil +} + +func (r *DevboxReconciler) syncProxyPod(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { + wsDeployment := &appsv1.Deployment{} + err := r.Client.Get(ctx, types.NamespacedName{Name: r.generateProxyPodDeploymentName(devbox), Namespace: devbox.Namespace}, wsDeployment) + if devbox.Spec.State == devboxv1alpha1.DevboxStateRunning { - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, wsPod, func() error { - return controllerutil.SetControllerReference(devbox, wsPod, r.Scheme) - }); err != nil { + if errors.IsNotFound(err) { + wsDeployment, err = r.generateProxyPodDeployment(ctx, devbox, recLabels, servicePorts) + if err != nil { + return err + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, wsDeployment, func() error { + return controllerutil.SetControllerReference(devbox, wsDeployment, r.Scheme) + }); err != nil { + return err + } + } else if err != nil { + return err + } + } else { + if err == nil { + if err := r.Client.Delete(ctx, wsDeployment); err != nil { + return err + } + } else if !errors.IsNotFound(err) { return err } } @@ -796,5 +946,7 @@ func (r *DevboxReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Pod{}, builder.WithPredicates(predicate.ResourceVersionChangedPredicate{})). // enqueue request if pod spec/status is updated Owns(&corev1.Service{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Owns(&corev1.Secret{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Owns(&networkingv1.Ingress{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Owns(&appsv1.Deployment{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Complete(r) } diff --git a/controllers/devbox/internal/controller/helper/devbox.go b/controllers/devbox/internal/controller/helper/devbox.go index b32b4d10f70..54fc27e6d82 100644 --- a/controllers/devbox/internal/controller/helper/devbox.go +++ b/controllers/devbox/internal/controller/helper/devbox.go @@ -77,7 +77,7 @@ func GenerateProxyPodLabels(devbox *devboxv1alpha1.Devbox, runtime *devboxv1alph } } recLabels := label.RecommendedLabels(&label.Recommended{ - Name: devbox.Name, + Name: devbox.Name + "-proxy", ManagedBy: label.DefaultManagedBy, PartOf: DevBoxPartOf, Component: ProxyPod, @@ -445,12 +445,12 @@ func GenerateResourceRequirements(devbox *devboxv1alpha1.Devbox, requestCPURate, func GenerateProxyPodResourceRequirements() corev1.ResourceRequirements { return corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("100Mi"), + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("100Mi"), + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("50Mi"), }, } }