diff --git a/controllers/devbox/api/v1alpha1/devbox_types.go b/controllers/devbox/api/v1alpha1/devbox_types.go index 5459881446e..d9a30d4df33 100644 --- a/controllers/devbox/api/v1alpha1/devbox_types.go +++ b/controllers/devbox/api/v1alpha1/devbox_types.go @@ -70,6 +70,14 @@ type NetworkSpec struct { ExtraPorts []corev1.ContainerPort `json:"extraPorts"` } +type AutoShutdownSpec struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + Enable bool `json:"type"` + // +kubebuilder:validation:Optional + Time string `json:"time"` +} + // DevboxSpec defines the desired state of Devbox type DevboxSpec struct { // +kubebuilder:validation:Required @@ -88,6 +96,9 @@ type DevboxSpec struct { // +kubebuilder:validation:Required NetworkSpec NetworkSpec `json:"network,omitempty"` + // +kubebuilder:validation:Optional + AutoShutdownSpec AutoShutdownSpec `json:"autoShutdown,omitempty"` + // todo add rewrite labels and annotations... // +kubebuilder:validation:Optional ExtraLabels map[string]string `json:"extraLabels,omitempty"` diff --git a/controllers/devbox/cmd/main.go b/controllers/devbox/cmd/main.go index fafa23264ab..0931cdd8191 100644 --- a/controllers/devbox/cmd/main.go +++ b/controllers/devbox/cmd/main.go @@ -74,7 +74,12 @@ func main() { var requestEphemeralStorage string var limitEphemeralStorage string var debugMode bool - var WebSocketImage string + var webSocketImage string + var websocketProxyDomain string + var ingressClass string + var enableAutoShutdown bool + var shutdownServerKey string + var shutdownServerAddr string flag.StringVar(®istryAddr, "registry-addr", "sealos.hub:5000", "The address of the registry") flag.StringVar(®istryUser, "registry-user", "admin", "The user of the registry") flag.StringVar(®istryPassword, "registry-password", "passw0rd", "The password of the registry") @@ -94,7 +99,12 @@ func main() { flag.Float64Var(&requestMemoryRate, "request-memory-rate", 10, "The request rate of memory limit in devbox.") flag.StringVar(&requestEphemeralStorage, "request-ephemeral-storage", "500Mi", "The request value of ephemeral storage in devbox.") flag.StringVar(&limitEphemeralStorage, "limit-ephemeral-storage", "10Gi", "The limit value of ephemeral storage in devbox.") - flag.StringVar(&WebSocketImage, "websocket-image", "bearslyricattack/chisel:1.0", "The image name of devbox websocket proxy pod") + flag.StringVar(&webSocketImage, "websocket-image", "cbluebird/wst:v0.0.4", "The image name of devbox websocket proxy pod.") + flag.StringVar(&websocketProxyDomain, "websocket-proxy-domain", "sealoshzh.site", "The websocket proxy domain of devbox ingress.") + flag.StringVar(&ingressClass, "ingress-class", "nginx", "The ingress class name.") + flag.BoolVar(&enableAutoShutdown, "enable-auto-shutdown", true, "If set, Devbox auto shutdown will be enabled.") + flag.StringVar(&shutdownServerKey, "shutdown-server-key", "sealos-devbox-shutdown", "The server key used to shutdown the server.") + flag.StringVar(&shutdownServerAddr, "shutdown-server-addr", "http://shutdown-service:8082", "The shutdown server address.") opts := zap.Options{ Development: true, } @@ -194,7 +204,12 @@ func main() { RequestEphemeralStorage: requestEphemeralStorage, LimitEphemeralStorage: limitEphemeralStorage, DebugMode: debugMode, - WebSocketImage: WebSocketImage, + WebSocketImage: webSocketImage, + WebsocketProxyDomain: websocketProxyDomain, + IngressClass: ingressClass, + EnableAutoShutdown: enableAutoShutdown, + ShutdownServerKey: shutdownServerKey, + ShutdownServerAddr: shutdownServerAddr, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Devbox") os.Exit(1) diff --git a/controllers/devbox/internal/controller/devbox_controller.go b/controllers/devbox/internal/controller/devbox_controller.go index 40284204206..b057ac6f2b4 100644 --- a/controllers/devbox/internal/controller/devbox_controller.go +++ b/controllers/devbox/internal/controller/devbox_controller.go @@ -19,17 +19,17 @@ package controller import ( "context" "fmt" + "strconv" "time" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" networkingv1 "k8s.io/api/networking/v1" - devboxv1alpha1 "github.com/labring/sealos/controllers/devbox/api/v1alpha1" - "github.com/labring/sealos/controllers/devbox/internal/controller/helper" - "github.com/labring/sealos/controllers/devbox/label" - + "github.com/golang-jwt/jwt/v5" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -39,6 +39,10 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" + devboxv1alpha1 "github.com/labring/sealos/controllers/devbox/api/v1alpha1" + "github.com/labring/sealos/controllers/devbox/internal/controller/helper" + "github.com/labring/sealos/controllers/devbox/label" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -56,7 +60,11 @@ type DevboxReconciler struct { LimitEphemeralStorage string WebSocketImage string DebugMode bool - + WebsocketProxyDomain string + IngressClass string + EnableAutoShutdown bool + ShutdownServerKey string + ShutdownServerAddr string client.Client Scheme *runtime.Scheme Recorder record.EventRecorder @@ -414,10 +422,10 @@ func (r *DevboxReconciler) getRuntime(ctx context.Context, devbox *devboxv1alpha return runtimecr, nil } -func (r *DevboxReconciler) syncNetwork(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string) error { +func (r *DevboxReconciler) getServicePort(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string) ([]corev1.ServicePort, error) { runtimecr, err := r.getRuntime(ctx, devbox) if err != nil { - return err + return nil, err } var servicePorts []corev1.ServicePort for _, port := range runtimecr.Spec.Config.Ports { @@ -429,7 +437,6 @@ func (r *DevboxReconciler) syncNetwork(ctx context.Context, devbox *devboxv1alph }) } if len(servicePorts) == 0 { - //use the default value servicePorts = []corev1.ServicePort{ { Name: "devbox-ssh-port", @@ -439,6 +446,14 @@ func (r *DevboxReconciler) syncNetwork(ctx context.Context, devbox *devboxv1alph }, } } + return servicePorts, nil +} + +func (r *DevboxReconciler) syncNetwork(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string) error { + servicePorts, err := r.getServicePort(ctx, devbox, recLabels) + if err != nil { + return err + } switch devbox.Spec.NetworkSpec.Type { case devboxv1alpha1.NetworkTypeNodePort: return r.syncNodePortNetwork(ctx, devbox, recLabels, servicePorts) @@ -449,6 +464,10 @@ func (r *DevboxReconciler) syncNetwork(ctx context.Context, devbox *devboxv1alph } func (r *DevboxReconciler) syncWebSocketNetwork(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { + devbox.Status.Network.Type = devboxv1alpha1.NetworkTypeWebSocket + if err := r.Status().Update(ctx, devbox); err != nil { + return err + } if err := r.syncPodSvc(ctx, devbox, recLabels, servicePorts); err != nil { return err } @@ -458,53 +477,66 @@ func (r *DevboxReconciler) syncWebSocketNetwork(ctx context.Context, devbox *dev if err := r.syncProxySvc(ctx, devbox, recLabels, servicePorts); err != nil { return err } - if err := r.syncProxyIngress(ctx, devbox); err != nil { + if hostName, err := r.syncProxyIngress(ctx, devbox); err != nil { return err + } else { + devbox.Status.Network.WebSocket = hostName } - devbox.Status.Network.Type = devboxv1alpha1.NetworkTypeWebSocket - devbox.Status.Network.WebSocket = devbox.Name + "-proxy-ingress" return r.Status().Update(ctx, devbox) } -func (r *DevboxReconciler) syncProxyIngress(ctx context.Context, devbox *devboxv1alpha1.Devbox) error { +func (r *DevboxReconciler) generateProxyIngressHost() string { + return rand.String(12) + "." + r.WebsocketProxyDomain +} + +func (r *DevboxReconciler) syncProxyIngress(ctx context.Context, devbox *devboxv1alpha1.Devbox) (string, error) { + host := r.generateProxyIngressHost() + pathType := networkingv1.PathTypePrefix - wsIngress := &networkingv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: devbox.Name + "-proxy-ingress", - Namespace: devbox.Namespace, + ingressPath := []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: &pathType, + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: devbox.Name + "-proxy-svc", + Port: networkingv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, }, - Spec: networkingv1.IngressSpec{ - Rules: []networkingv1.IngressRule{ - { - Host: devbox.Name + ".sealoshzh.site", - IngressRuleValue: networkingv1.IngressRuleValue{ - HTTP: &networkingv1.HTTPIngressRuleValue{ - Paths: []networkingv1.HTTPIngressPath{ - { - Path: "/", - PathType: &pathType, - Backend: networkingv1.IngressBackend{ - Service: &networkingv1.IngressServiceBackend{ - Name: devbox.Name + "-proxy-svc", - Port: networkingv1.ServiceBackendPort{ - Number: 22, - }, - }, - }, - }, - }, - }, + } + + ingressSpec := networkingv1.IngressSpec{ + IngressClassName: &r.IngressClass, + Rules: []networkingv1.IngressRule{ + { + Host: host, + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: ingressPath, }, }, }, }, } + + wsIngress := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: devbox.Name + "-proxy-ingress", + Namespace: devbox.Namespace, + }, + Spec: ingressSpec, + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, wsIngress, func() error { return controllerutil.SetControllerReference(devbox, wsIngress, r.Scheme) }); err != nil { - return err + return "", err } - return nil + + return host, nil } func (r *DevboxReconciler) syncProxySvc(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { @@ -512,11 +544,18 @@ func (r *DevboxReconciler) syncProxySvc(ctx context.Context, devbox *devboxv1alp if err != nil { return err } - + servicePort := []corev1.ServicePort{ + { + Name: "devbox-ssh-port", + Port: 80, + TargetPort: intstr.FromInt32(80), + Protocol: corev1.ProtocolTCP, + }, + } expectServiceSpec := corev1.ServiceSpec{ Selector: helper.GenerateProxyPodLabels(devbox, runtimecr), Type: corev1.ServiceTypeClusterIP, - Ports: servicePorts, + Ports: servicePort, } proxySvc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -524,20 +563,15 @@ func (r *DevboxReconciler) syncProxySvc(ctx context.Context, devbox *devboxv1alp Namespace: devbox.Namespace, Labels: helper.GenerateProxyPodLabels(devbox, runtimecr), }, + Spec: expectServiceSpec, } - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, proxySvc, func() error { - // only update some specific fields proxySvc.Spec.Selector = expectServiceSpec.Selector proxySvc.Spec.Type = expectServiceSpec.Type - if len(proxySvc.Spec.Ports) == 0 { - proxySvc.Spec.Ports = expectServiceSpec.Ports - } else { - proxySvc.Spec.Ports[0].Name = expectServiceSpec.Ports[0].Name - proxySvc.Spec.Ports[0].Port = expectServiceSpec.Ports[0].Port - proxySvc.Spec.Ports[0].TargetPort = expectServiceSpec.Ports[0].TargetPort - proxySvc.Spec.Ports[0].Protocol = expectServiceSpec.Ports[0].Protocol - } + proxySvc.Spec.Ports[0].Name = expectServiceSpec.Ports[0].Name + proxySvc.Spec.Ports[0].Port = expectServiceSpec.Ports[0].Port + proxySvc.Spec.Ports[0].TargetPort = expectServiceSpec.Ports[0].TargetPort + proxySvc.Spec.Ports[0].Protocol = expectServiceSpec.Ports[0].Protocol return controllerutil.SetControllerReference(devbox, proxySvc, r.Scheme) }); err != nil { return err @@ -545,12 +579,63 @@ func (r *DevboxReconciler) syncProxySvc(ctx context.Context, devbox *devboxv1alp return nil } -func (r *DevboxReconciler) syncProxyPod(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { - runtimecr, err := r.getRuntime(ctx, devbox) +func (r *DevboxReconciler) generateProxyPodName(devbox *devboxv1alpha1.Devbox) string { + return devbox.Name + "-proxy-pod" + "-" + rand.String(5) +} + +func (r *DevboxReconciler) generateProxyPodDeploymentName(devbox *devboxv1alpha1.Devbox) string { + return devbox.Name + "-proxy-deployment" +} + +type DevboxClaims struct { + DevboxName string `json:"devbox_name"` + NameSpace string `json:"namespace"` + jwt.RegisteredClaims +} + +func (r *DevboxReconciler) generateProxyPodJWT(ctx context.Context, devbox *devboxv1alpha1.Devbox) (string, error) { + claims := DevboxClaims{ + DevboxName: devbox.Name, + NameSpace: devbox.Namespace, + RegisteredClaims: jwt.RegisteredClaims{ + IssuedAt: jwt.NewNumericDate(time.Now()), + ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour * 7 * 24)), + Issuer: "devbox-controller", + }, + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + signedToken, err := token.SignedString([]byte(r.ShutdownServerKey)) if err != nil { - return err + return "", fmt.Errorf("failed to sign token: %w", err) + } + return signedToken, nil +} + +func (r *DevboxReconciler) generateProxyPodEnv(ctx context.Context, devbox *devboxv1alpha1.Devbox, servicePorts []corev1.ServicePort) ([]corev1.EnvVar, error) { + var envVars []corev1.EnvVar + autoShutdownEnabled := devbox.Spec.AutoShutdownSpec.Enable && r.EnableAutoShutdown + envVars = append(envVars, corev1.EnvVar{ + Name: "ENABLE_AUTO_SHUTDOWN", + Value: strconv.FormatBool(autoShutdownEnabled), + }) + + if autoShutdownEnabled { + envVars = append(envVars, corev1.EnvVar{ + Name: "AUTO_SHUTDOWN_INTERVAL", + Value: devbox.Spec.AutoShutdownSpec.Time, + }) } + token, err := r.generateProxyPodJWT(ctx, devbox) + if err != nil { + return nil, err + } + + envVars = append(envVars, corev1.EnvVar{ + Name: "JWT_TOKEN", + Value: token, + }) + sshPort := "22" for _, port := range servicePorts { if port.Name == "devbox-ssh-port" { @@ -558,44 +643,96 @@ func (r *DevboxReconciler) syncProxyPod(ctx context.Context, devbox *devboxv1alp break } } + envVars = append(envVars, corev1.EnvVar{ + Name: "TARGET", + Value: fmt.Sprintf("%s-pod-svc:%s", devbox.Name, sshPort), + }) - podName := devbox.Name + "-proxy-pod" - wsPod := &corev1.Pod{} - err = r.Client.Get(ctx, types.NamespacedName{Name: podName, Namespace: devbox.Namespace}, wsPod) - if err == nil { - return nil + envVars = append(envVars, corev1.EnvVar{ + Name: "LISTEN", + Value: "0.0.0.0:80", + }) + + envVars = append(envVars, corev1.EnvVar{ + Name: "AUTO_SHUTDOWN_SERVICE_URL", + Value: r.ShutdownServerAddr, + }) + + return envVars, nil +} + +func (r *DevboxReconciler) generateProxyPodDeployment(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) (*appsv1.Deployment, error) { + runtimecr, err := r.getRuntime(ctx, devbox) + if err != nil { + return nil, err } - if !errors.IsNotFound(err) { - return err + + podEnv, err := r.generateProxyPodEnv(ctx, devbox, servicePorts) + if err != nil { + return nil, err } - wsPod = &corev1.Pod{ + + podSpec := corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "ws-proxy", + Image: r.WebSocketImage, + Env: podEnv, + Resources: helper.GenerateProxyPodResourceRequirements(), + }, + }, + } + + replicas := int32(1) + return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: podName, + Name: r.generateProxyPodDeploymentName(devbox), Namespace: devbox.Namespace, Labels: helper.GenerateProxyPodLabels(devbox, runtimecr), Annotations: helper.GeneratePodAnnotations(devbox, runtimecr), }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "ws-proxy", - Image: r.WebSocketImage, - Args: []string{ - "server", - fmt.Sprintf("--port=%d", 8080), - fmt.Sprintf("--proxy=%s", "https://"+devbox.Name+"-pod-svc"+sshPort), - "-v=true", - "--reverse", - }, - Resources: helper.GenerateProxyPodResourceRequirements(), + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: helper.GenerateProxyPodLabels(devbox, runtimecr), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.generateProxyPodName(devbox), + Namespace: devbox.Namespace, + Labels: helper.GenerateProxyPodLabels(devbox, runtimecr), + Annotations: helper.GeneratePodAnnotations(devbox, runtimecr), }, + Spec: podSpec, }, }, - } + }, nil +} + +func (r *DevboxReconciler) syncProxyPod(ctx context.Context, devbox *devboxv1alpha1.Devbox, recLabels map[string]string, servicePorts []corev1.ServicePort) error { + wsDeployment := &appsv1.Deployment{} + err := r.Client.Get(ctx, types.NamespacedName{Name: r.generateProxyPodDeploymentName(devbox), Namespace: devbox.Namespace}, wsDeployment) + if devbox.Spec.State == devboxv1alpha1.DevboxStateRunning { - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, wsPod, func() error { - return controllerutil.SetControllerReference(devbox, wsPod, r.Scheme) - }); err != nil { + if errors.IsNotFound(err) { + wsDeployment, err = r.generateProxyPodDeployment(ctx, devbox, recLabels, servicePorts) + if err != nil { + return err + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, wsDeployment, func() error { + return controllerutil.SetControllerReference(devbox, wsDeployment, r.Scheme) + }); err != nil { + return err + } + } else if err != nil { + return err + } + } else { + if err == nil { + if err := r.Client.Delete(ctx, wsDeployment); err != nil { + return err + } + } else if !errors.IsNotFound(err) { return err } } @@ -809,5 +946,7 @@ func (r *DevboxReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Pod{}, builder.WithPredicates(predicate.ResourceVersionChangedPredicate{})). // enqueue request if pod spec/status is updated Owns(&corev1.Service{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Owns(&corev1.Secret{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Owns(&networkingv1.Ingress{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Owns(&appsv1.Deployment{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Complete(r) } diff --git a/controllers/devbox/internal/controller/helper/devbox.go b/controllers/devbox/internal/controller/helper/devbox.go index eab785e878b..54fc27e6d82 100644 --- a/controllers/devbox/internal/controller/helper/devbox.go +++ b/controllers/devbox/internal/controller/helper/devbox.go @@ -445,12 +445,12 @@ func GenerateResourceRequirements(devbox *devboxv1alpha1.Devbox, requestCPURate, func GenerateProxyPodResourceRequirements() corev1.ResourceRequirements { return corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("100Mi"), + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("100Mi"), + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("50Mi"), }, } }