feat(infra): introduce public IP to node info

.github/workflows/build-docker-image.yaml

@@ -3,7 +3,7 @@ name: Build Docker Image
 on:
   workflow_call:
     inputs:
-      folder:
+      dockerfile:
         required: true
         type: string
       repo:
@@ -49,7 +49,7 @@ jobs:
         run: |
           set -x
 
-          docker build . --file ${{ inputs.folder }}/Dockerfile \
+          docker build . --file ${{ inputs.dockerfile }} \
             --tag ${{ inputs.repo }}:${{ inputs.tag }} \
             --build-arg GIT_COMMIT=${{ inputs.commit }} \
             --build-arg REPO=$ECR_REGISTRY/ecr-public

.github/workflows/pr.yaml

@@ -0,0 +1,20 @@
+name: pr
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/pr.yaml"
+      - ".github/workflows/build-docer-image.yaml"
+      - "pkg/**"
+      - "cmd/**"
+      - "hack/dockerfiles/**"
+    branches: ["**"]
+
+jobs:
+  build:
+    uses: ./.github/workflows/build-docker-image.yaml
+    secrets: inherit
+    with:
+      dockerfile: "hack/dockerfiles/release/Dockerfile.yurt-manager"
+      repo: raven-manager
+

.github/workflows/push.yaml

@@ -0,0 +1,20 @@
+name: push
+
+on:
+  push:
+    branches: ["main"]
+
+jobs:
+  build:
+    uses: ./.github/workflows/build-docker-image.yaml
+    secrets: inherit
+    with:
+      dockerfile: "hack/dockerfiles/release/Dockerfile.yurt-manager"
+      repo: raven-manager
+      tag: latest
+
+
+
+
+
+

.github/workflows/release.yaml

@@ -0,0 +1,17 @@
+name: release
+
+on:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  build:
+    uses: ./.github/workflows/build-docker-image.yaml
+    secrets: inherit
+    with:
+      dockerfile: "hack/dockerfiles/release/Dockerfile.yurt-manager"
+      repo: raven-manager
+      tag: ${GITHUB_REF##*/}
+      commit: ${GITHUB_REF##*/}
+

charts/yurt-manager/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.3.4
+version: 1.3.5
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.3.4"
+appVersion: "1.3.5"

charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml

@@ -123,6 +123,9 @@ spec:
                         type: string
                       privateIP:
                         type: string
+                      publicIP:
+                        description: 'Node PublicIP: satellite nodes need to build vxlan connctions using their public IPs;'
+                        type: string
                       subnets:
                         items:
                           type: string
@@ -285,7 +288,10 @@ spec:
                         description: NodeName is the Node host name.
                         type: string
                       privateIP:
-                        description: PrivateIP is the node private ip address
+                        description: PrivateIP is the node private ip address in lepton cluster
+                        type: string
+                      publicIP:
+                        description: 'Node PublicIP: satellite nodes need to build vxlan connctions using their public IPs;'
                         type: string
                       subnets:
                         description: Subnets is the pod ip range of the node

charts/yurt-manager/templates/yurt-manager.yaml

@@ -80,7 +80,7 @@ spec:
           command:
             - /usr/local/bin/yurt-manager
           image: {{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
           name: yurt-manager
           ports:
             - containerPort: {{ .Values.ports.webhook }}

charts/yurt-manager/values.yaml

@@ -14,6 +14,7 @@ image:
   registry: openyurt
   repository: yurt-manager
   tag: v1.3.4
+  pullPolicy: Always
 
 ports:
   metrics: 10271

pkg/apis/raven/v1alpha1/gateway_conversion.go

@@ -62,6 +62,7 @@ func (src *Gateway) ConvertTo(dstRaw conversion.Hub) error {
 			NodeName:  node.NodeName,
 			PrivateIP: node.PrivateIP,
 			Subnets:   node.Subnets,
+			PublicIP:  node.PublicIP,
 		})
 	}
 	if src.Status.ActiveEndpoint != nil {
@@ -102,6 +103,7 @@ func (dst *Gateway) ConvertFrom(srcRaw conversion.Hub) error {
 			NodeName:  node.NodeName,
 			PrivateIP: node.PrivateIP,
 			Subnets:   node.Subnets,
+			PublicIP:  node.PublicIP,
 		})
 	}
 	if src.Status.ActiveEndpoints == nil {

pkg/apis/raven/v1alpha1/gateway_types.go

@@ -70,6 +70,9 @@ type NodeInfo struct {
 	NodeName  string   `json:"nodeName"`
 	PrivateIP string   `json:"privateIP"`
 	Subnets   []string `json:"subnets"`
+	// Node PublicIP: satellite nodes need to build vxlan connctions
+	// using their public IPs;
+	PublicIP string `json:"publicIP,omitempty"`
 }
 
 // GatewayStatus defines the observed state of Gateway

pkg/apis/raven/v1beta1/gateway_types.go

@@ -95,10 +95,13 @@ type Endpoint struct {
 type NodeInfo struct {
 	// NodeName is the Node host name.
 	NodeName string `json:"nodeName"`
-	// PrivateIP is the node private ip address
+	// PrivateIP is the node private ip address in lepton cluster
 	PrivateIP string `json:"privateIP"`
 	// Subnets is the pod ip range of the node
 	Subnets []string `json:"subnets"`
+	// Node PublicIP: satellite nodes need to build vxlan connctions
+	// using their public IPs;
+	PublicIP string `json:"publicIP,omitempty"`
 }
 
 // GatewayStatus defines the observed state of Gateway

pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go

@@ -177,9 +177,10 @@ func (r *ReconcileGateway) Reconcile(ctx context.Context, req reconcile.Request)
 		err = fmt.Errorf("unable to list nodes: %s", err)
 		return reconcile.Result{}, err
 	}
-	klog.V(1).Info(Format("list gateway %d node %v", len(nodeList.Items), nodeList.Items))
+	klog.V(1).Info(Format("list gateway %d", len(nodeList.Items)))
 	// 1. try to elect an active endpoint if possible
 	activeEp := r.electActiveEndpoint(nodeList, &gw)
+	klog.V(1).Infof("Elected %d endpoint for gateway %s: %v", len(activeEp), gw.Name, activeEp)
 	r.recordEndpointEvent(&gw, gw.Status.ActiveEndpoints, activeEp)
 	gw.Status.ActiveEndpoints = activeEp
 	r.configEndpoints(ctx, &gw)
@@ -191,10 +192,16 @@ func (r *ReconcileGateway) Reconcile(ctx context.Context, req reconcile.Request)
 			klog.ErrorS(err, "unable to get podCIDR")
 			return reconcile.Result{}, err
 		}
+		publicIP := ""
+		publicIP, err = utils.GetEdgeNodePublicIP(&v)
+		if err != nil {
+			klog.InfoS("unable to get node public IP, expecting nodes to communicate via private IPs: " + err.Error())
+		}
 		nodes = append(nodes, ravenv1beta1.NodeInfo{
 			NodeName:  v.Name,
 			PrivateIP: utils.GetNodeInternalIP(v),
 			Subnets:   podCIDRs,
+			PublicIP:  publicIP,
 		})
 	}
 	sort.Slice(nodes, func(i, j int) bool { return nodes[i].NodeName < nodes[j].NodeName })
@@ -249,10 +256,11 @@ func (r *ReconcileGateway) electActiveEndpoint(nodeList corev1.NodeList, gw *rav
 			readyNodes[v.Name] = &v
 		}
 	}
-	klog.V(1).Infof(Format("Ready node has %d, node %v", len(readyNodes), readyNodes))
+	klog.V(1).Infof(Format("Ready node has %d", len(readyNodes)))
 	// init a endpoints slice
 	enableProxy, enableTunnel := utils.CheckServer(context.TODO(), r.Client)
 	eps := make([]*ravenv1beta1.Endpoint, 0)
+	klog.V(1).Info("enableProxy=%t, enableTunnel=%t", enableProxy, enableTunnel)
 	if enableProxy {
 		eps = append(eps, electEndpoints(gw, ravenv1beta1.Proxy, readyNodes)...)
 	}
@@ -264,6 +272,7 @@ func (r *ReconcileGateway) electActiveEndpoint(nodeList corev1.NodeList, gw *rav
 }
 
 func electEndpoints(gw *ravenv1beta1.Gateway, endpointType string, readyNodes map[string]*corev1.Node) []*ravenv1beta1.Endpoint {
+	klog.V(1).Infof("Electing endpoints for gateway %s with endpoint type %s", gw.Name, endpointType)
 	eps := make([]*ravenv1beta1.Endpoint, 0)
 	var replicas int
 	switch endpointType {

pkg/yurtmanager/controller/raven/utils/utils.go

@@ -40,6 +40,7 @@ const (
 	WorkingNamespace               = "kube-system"
 	RavenGlobalConfig              = "raven-cfg"
 	LabelCurrentGatewayEndpoints   = "raven.openyurt.io/endpoints-name"
+	LabelNodeProviderPublicIP      = "raven.openyurt.io/provider-public-ip"
 	GatewayProxyInternalService    = "x-raven-proxy-internal-svc"
 	GatewayProxyServiceNamePrefix  = "x-raven-proxy-svc"
 	GatewayTunnelServiceNamePrefix = "x-raven-tunnel-svc"
@@ -53,6 +54,8 @@ const (
 	VPNServerExposedPortKey    = "tunnel-bind-addr"
 	RavenEnableProxy           = "enable-l7-proxy"
 	RavenEnableTunnel          = "enable-l3-tunnel"
+	DefaultEnableL7Proxy       = false
+	DefaultEnableL3Tunnel      = true
 )
 
 // GetNodeInternalIP returns internal ip of the given `node`.
@@ -67,6 +70,17 @@ func GetNodeInternalIP(node corev1.Node) string {
 	return ip
 }
 
+func GetEdgeNodePublicIP(node *corev1.Node) (string, error) {
+	ip, ok := node.Labels[LabelNodeProviderPublicIP]
+	if !ok {
+		return "", fmt.Errorf("failed to get public ip, no label %s on node %s", LabelNodeProviderPublicIP, node.Name)
+	}
+	if net.ParseIP(ip) == nil {
+		return "", fmt.Errorf("failed to get public ip, invalid public IP label %s, %s on node %s", LabelNodeProviderPublicIP, ip, node.Name)
+	}
+	return ip, nil
+}
+
 // AddGatewayToWorkQueue adds the Gateway the reconciler's workqueue
 func AddGatewayToWorkQueue(gwName string,
 	q workqueue.RateLimitingInterface) {
@@ -79,8 +93,8 @@ func AddGatewayToWorkQueue(gwName string,
 
 func CheckServer(ctx context.Context, client client.Client) (enableProxy, enableTunnel bool) {
 	var cm corev1.ConfigMap
-	enableTunnel = false
-	enableProxy = false
+	enableTunnel = DefaultEnableL3Tunnel
+	enableProxy = DefaultEnableL7Proxy
 	err := client.Get(ctx, types.NamespacedName{Namespace: WorkingNamespace, Name: RavenGlobalConfig}, &cm)
 	if err != nil {
 		return enableProxy, enableTunnel