From 9ab18062237801f59bb16c5625bdb1ef8ea4eeb2 Mon Sep 17 00:00:00 2001 From: suyuee Date: Thu, 12 Oct 2023 15:06:22 -0700 Subject: [PATCH 1/9] feat(infra): introduce public IP --- ...lockaffinities.apiregistration.k8s.io.yaml | 62 +++++++++++++++++++ .../crds/raven.openyurt.io_gateways.yaml | 6 +- pkg/apis/calico/v3/zz_generated.deepcopy.go | 26 ++------ pkg/apis/raven/v1beta1/gateway_types.go | 5 +- .../gateway_pickup_controller.go | 7 +++ .../controller/raven/utils/utils.go | 12 ++++ 6 files changed, 94 insertions(+), 24 deletions(-) create mode 100644 charts/yurt-manager/crds/apiextensions.k8s.io_v1_customresourcedefinition_blockaffinities.apiregistration.k8s.io.yaml diff --git a/charts/yurt-manager/crds/apiextensions.k8s.io_v1_customresourcedefinition_blockaffinities.apiregistration.k8s.io.yaml b/charts/yurt-manager/crds/apiextensions.k8s.io_v1_customresourcedefinition_blockaffinities.apiregistration.k8s.io.yaml new file mode 100644 index 00000000000..ab1b7eb8077 --- /dev/null +++ b/charts/yurt-manager/crds/apiextensions.k8s.io_v1_customresourcedefinition_blockaffinities.apiregistration.k8s.io.yaml @@ -0,0 +1,62 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: blockaffinities.apiregistration.k8s.io +spec: + group: apiregistration.k8s.io + names: + kind: BlockAffinity + listKind: BlockAffinityList + plural: blockaffinities + singular: blockaffinity + scope: Namespaced + versions: + - name: v3 + schema: + openAPIV3Schema: + description: BlockAffinity maintains a block affinity's state + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of the BlockAffinity. + properties: + cidr: + type: string + deleted: + description: Deleted indicates that this block affinity is being deleted. + This field is a string for compatibility with older releases that + mistakenly treat this field as a string. + type: string + node: + type: string + state: + type: string + required: + - cidr + - deleted + - node + - state + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml b/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml index c3a5d289129..aef214cb2d6 100644 --- a/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml +++ b/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml @@ -285,7 +285,10 @@ spec: description: NodeName is the Node host name. type: string privateIP: - description: PrivateIP is the node private ip address + description: PrivateIP is the node private ip address in lepton cluster + type: string + publicIP: + description: 'Node PublicIP: satellite nodes need to build vxlan connctions using their public IPs;' type: string subnets: description: Subnets is the pod ip range of the node @@ -295,6 +298,7 @@ spec: required: - nodeName - privateIP + - publicIP - subnets type: object type: array diff --git a/pkg/apis/calico/v3/zz_generated.deepcopy.go b/pkg/apis/calico/v3/zz_generated.deepcopy.go index dd0489c4a0d..22a0931aa07 100644 --- a/pkg/apis/calico/v3/zz_generated.deepcopy.go +++ b/pkg/apis/calico/v3/zz_generated.deepcopy.go @@ -4,25 +4,25 @@ /* Copyright 2023 The OpenYurt Authors. -Licensed under the Apache License, Version 2.0 (the License); +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an AS IS BASIS, +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -// Code generated by deepcopy-gen. DO NOT EDIT. +// Code generated by controller-gen. DO NOT EDIT. package v3 import ( - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -31,7 +31,6 @@ func (in *BlockAffinity) DeepCopyInto(out *BlockAffinity) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockAffinity. @@ -64,7 +63,6 @@ func (in *BlockAffinityList) DeepCopyInto(out *BlockAffinityList) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockAffinityList. @@ -84,19 +82,3 @@ func (in *BlockAffinityList) DeepCopyObject() runtime.Object { } return nil } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BlockAffinitySpec) DeepCopyInto(out *BlockAffinitySpec) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockAffinitySpec. -func (in *BlockAffinitySpec) DeepCopy() *BlockAffinitySpec { - if in == nil { - return nil - } - out := new(BlockAffinitySpec) - in.DeepCopyInto(out) - return out -} diff --git a/pkg/apis/raven/v1beta1/gateway_types.go b/pkg/apis/raven/v1beta1/gateway_types.go index a8d433c0a46..43eef1e298b 100644 --- a/pkg/apis/raven/v1beta1/gateway_types.go +++ b/pkg/apis/raven/v1beta1/gateway_types.go @@ -95,10 +95,13 @@ type Endpoint struct { type NodeInfo struct { // NodeName is the Node host name. NodeName string `json:"nodeName"` - // PrivateIP is the node private ip address + // PrivateIP is the node private ip address in lepton cluster PrivateIP string `json:"privateIP"` // Subnets is the pod ip range of the node Subnets []string `json:"subnets"` + // Node PublicIP: satellite nodes need to build vxlan connctions + // using their public IPs; + PublicIP string `json:"publicIP"` } // GatewayStatus defines the observed state of Gateway diff --git a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go index 055841e6297..790af346a24 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go @@ -191,10 +191,17 @@ func (r *ReconcileGateway) Reconcile(ctx context.Context, req reconcile.Request) klog.ErrorS(err, "unable to get podCIDR") return reconcile.Result{}, err } + publicIP := "" + publicIP, err = utils.GetLeptonSatelliteNodePublicIP(v) + if err != nil { + klog.ErrorS(err, "unable to get node public IP") + return reconcile.Result{}, err + } nodes = append(nodes, ravenv1beta1.NodeInfo{ NodeName: v.Name, PrivateIP: utils.GetNodeInternalIP(v), Subnets: podCIDRs, + PublicIP: publicIP, }) } sort.Slice(nodes, func(i, j int) bool { return nodes[i].NodeName < nodes[j].NodeName }) diff --git a/pkg/yurtmanager/controller/raven/utils/utils.go b/pkg/yurtmanager/controller/raven/utils/utils.go index b47ad097f41..8f3ae83e631 100644 --- a/pkg/yurtmanager/controller/raven/utils/utils.go +++ b/pkg/yurtmanager/controller/raven/utils/utils.go @@ -40,6 +40,7 @@ const ( WorkingNamespace = "kube-system" RavenGlobalConfig = "raven-cfg" LabelCurrentGatewayEndpoints = "raven.openyurt.io/endpoints-name" + LabelLeptonSatellitePublicIP = "lepton.ai/provider-public-ip" GatewayProxyInternalService = "x-raven-proxy-internal-svc" GatewayProxyServiceNamePrefix = "x-raven-proxy-svc" GatewayTunnelServiceNamePrefix = "x-raven-tunnel-svc" @@ -67,6 +68,17 @@ func GetNodeInternalIP(node corev1.Node) string { return ip } +func GetLeptonSatelliteNodePublicIP(node corev1.Node) (string, error) { + ip, ok := node.Labels[LabelLeptonSatellitePublicIP] + if !ok { + return "", fmt.Errorf("Failed to get public ip, no label %s on node %s", LabelLeptonSatellitePublicIP, node.Name) + } + if net.ParseIP(ip) == nil { + return "", fmt.Errorf("Failed to get public ip, invalid public IP label %s, %s on node %s", LabelLeptonSatellitePublicIP, ip, node.Name) + } + return ip, nil +} + // AddGatewayToWorkQueue adds the Gateway the reconciler's workqueue func AddGatewayToWorkQueue(gwName string, q workqueue.RateLimitingInterface) { From b165e0d7d73ab70fe2cd4f5223f16c8a1d68f516 Mon Sep 17 00:00:00 2001 From: suyuee Date: Thu, 12 Oct 2023 22:21:28 -0700 Subject: [PATCH 2/9] workflow files --- .github/workflows/build-docer-image.yaml | 59 ++++++++++++++++++++++++ .github/workflows/pr.yaml | 16 +++++++ .github/workflows/push.yaml | 22 +++++++++ .github/workflows/release.yaml | 17 +++++++ 4 files changed, 114 insertions(+) create mode 100644 .github/workflows/build-docer-image.yaml create mode 100644 .github/workflows/pr.yaml create mode 100644 .github/workflows/push.yaml create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/build-docer-image.yaml b/.github/workflows/build-docer-image.yaml new file mode 100644 index 00000000000..8e1755f2401 --- /dev/null +++ b/.github/workflows/build-docer-image.yaml @@ -0,0 +1,59 @@ +name: Build Docker Image + +on: + workflow_call: + inputs: + folder: + required: true + type: string + repo: + required: true + type: string + tag: + required: false + type: string + default: test$(git rev-parse --short HEAD) + commit: + required: false + type: string + default: $(git rev-parse --short HEAD) + runs-on: + required: false + type: string + default: ubuntu-latest + +jobs: + build: + runs-on: ${{ inputs.runs-on }} + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ secrets.SUBMODULES_GITHUB_TOKEN }} + - name: Install awscli + run: | + python -m pip install --upgrade pip + pip install awscli + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: Build and push the Docker image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + set -x + + docker build . --file ${{ inputs.folder }}/Dockerfile \ + --tag ${{ inputs.repo }}:${{ inputs.tag }} \ + --build-arg GIT_COMMIT=${{ inputs.commit }} \ + --build-arg REPO=$ECR_REGISTRY/ecr-public + + echo "tagging container image with ${{ inputs.repo }}:${{ inputs.tag }}}" + docker tag ${{ inputs.repo }}:${{ inputs.tag }} $ECR_REGISTRY/${{ inputs.repo }}:${{ inputs.tag }} + docker push $ECR_REGISTRY/${{ inputs.repo }}:${{ inputs.tag }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 00000000000..44c84dd80a3 --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,16 @@ +name: pr + +on: + pull_request: + paths: + - "**" + branches: ["**"] + +jobs: + build: + uses: ./.github/workflows/build-docker-image.yaml + secrets: inherit + with: + folder: ./ + repo: raven-manager + diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml new file mode 100644 index 00000000000..23f6df38c33 --- /dev/null +++ b/.github/workflows/push.yaml @@ -0,0 +1,22 @@ +name: push + +on: + push: + paths: + - "**" + branches: ["main"] + +jobs: + build: + uses: ./.github/workflows/build-docker-image.yaml + secrets: inherit + with: + folder: ./ + repo: raven-manager + tag: latest + + + + + + diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000000..eec354993d7 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,17 @@ +name: release + +on: + push: + tags: + - "*" + +jobs: + build: + uses: ./.github/workflows/build-docker-image.yaml + secrets: inherit + with: + folder: ./ + repo: raven-manager + tag: ${GITHUB_REF##*/} + commit: ${GITHUB_REF##*/} + From eb4273327da9d8136ada58e21b45db20761440a6 Mon Sep 17 00:00:00 2001 From: suyuee Date: Thu, 12 Oct 2023 22:31:49 -0700 Subject: [PATCH 3/9] empty --- .github/workflows/build-docer-image.yaml | 59 ------------------------ 1 file changed, 59 deletions(-) delete mode 100644 .github/workflows/build-docer-image.yaml diff --git a/.github/workflows/build-docer-image.yaml b/.github/workflows/build-docer-image.yaml deleted file mode 100644 index 8e1755f2401..00000000000 --- a/.github/workflows/build-docer-image.yaml +++ /dev/null @@ -1,59 +0,0 @@ -name: Build Docker Image - -on: - workflow_call: - inputs: - folder: - required: true - type: string - repo: - required: true - type: string - tag: - required: false - type: string - default: test$(git rev-parse --short HEAD) - commit: - required: false - type: string - default: $(git rev-parse --short HEAD) - runs-on: - required: false - type: string - default: ubuntu-latest - -jobs: - build: - runs-on: ${{ inputs.runs-on }} - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ secrets.SUBMODULES_GITHUB_TOKEN }} - - name: Install awscli - run: | - python -m pip install --upgrade pip - pip install awscli - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Build and push the Docker image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - run: | - set -x - - docker build . --file ${{ inputs.folder }}/Dockerfile \ - --tag ${{ inputs.repo }}:${{ inputs.tag }} \ - --build-arg GIT_COMMIT=${{ inputs.commit }} \ - --build-arg REPO=$ECR_REGISTRY/ecr-public - - echo "tagging container image with ${{ inputs.repo }}:${{ inputs.tag }}}" - docker tag ${{ inputs.repo }}:${{ inputs.tag }} $ECR_REGISTRY/${{ inputs.repo }}:${{ inputs.tag }} - docker push $ECR_REGISTRY/${{ inputs.repo }}:${{ inputs.tag }} From 9e37aba5de5a50339688065b8829e1da01faf157 Mon Sep 17 00:00:00 2001 From: suyuee Date: Sun, 15 Oct 2023 17:00:05 -0700 Subject: [PATCH 4/9] build workflow --- .github/workflows/build-docker-image.yaml | 4 ++-- .github/workflows/pr.yaml | 9 +++++++-- .github/workflows/release.yaml | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-docker-image.yaml b/.github/workflows/build-docker-image.yaml index 8e1755f2401..5f2811917c3 100644 --- a/.github/workflows/build-docker-image.yaml +++ b/.github/workflows/build-docker-image.yaml @@ -3,7 +3,7 @@ name: Build Docker Image on: workflow_call: inputs: - folder: + dockerfile: required: true type: string repo: @@ -49,7 +49,7 @@ jobs: run: | set -x - docker build . --file ${{ inputs.folder }}/Dockerfile \ + docker build . --file ${{ inputs.dockerfile }} \ --tag ${{ inputs.repo }}:${{ inputs.tag }} \ --build-arg GIT_COMMIT=${{ inputs.commit }} \ --build-arg REPO=$ECR_REGISTRY/ecr-public diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 44c84dd80a3..3368ef5a6b1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -3,7 +3,12 @@ name: pr on: pull_request: paths: - - "**" + - ".github/workflows/pr.yaml" + - ".github/workflows/build-docer-image.yaml" + - "pkg/**" + - "cmd/**" + - "charts/**" + - "hack/dockerfiles/**" branches: ["**"] jobs: @@ -11,6 +16,6 @@ jobs: uses: ./.github/workflows/build-docker-image.yaml secrets: inherit with: - folder: ./ + dockerfile: "hack/dockerfiles/release/Dockerfile.yurt-manager" repo: raven-manager diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index eec354993d7..bb3787ba3a7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,7 +10,7 @@ jobs: uses: ./.github/workflows/build-docker-image.yaml secrets: inherit with: - folder: ./ + dockerfile: "hack/dockerfiles/release/Dockerfile.yurt-manager" repo: raven-manager tag: ${GITHUB_REF##*/} commit: ${GITHUB_REF##*/} From 534d6a12cddcce91fd15687748cbb820f90840ed Mon Sep 17 00:00:00 2001 From: suyuee Date: Sun, 15 Oct 2023 20:13:41 -0700 Subject: [PATCH 5/9] comments --- .github/workflows/pr.yaml | 1 - .github/workflows/push.yaml | 4 +--- .../raven/gatewaypickup/gateway_pickup_controller.go | 2 +- pkg/yurtmanager/controller/raven/utils/utils.go | 8 ++++---- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3368ef5a6b1..78c591787c2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -7,7 +7,6 @@ on: - ".github/workflows/build-docer-image.yaml" - "pkg/**" - "cmd/**" - - "charts/**" - "hack/dockerfiles/**" branches: ["**"] diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 23f6df38c33..998d0c92b3b 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -2,8 +2,6 @@ name: push on: push: - paths: - - "**" branches: ["main"] jobs: @@ -11,7 +9,7 @@ jobs: uses: ./.github/workflows/build-docker-image.yaml secrets: inherit with: - folder: ./ + dockerfile: "hack/dockerfiles/release/Dockerfile.yurt-manager" repo: raven-manager tag: latest diff --git a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go index 790af346a24..c13d133a381 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go @@ -192,7 +192,7 @@ func (r *ReconcileGateway) Reconcile(ctx context.Context, req reconcile.Request) return reconcile.Result{}, err } publicIP := "" - publicIP, err = utils.GetLeptonSatelliteNodePublicIP(v) + publicIP, err = utils.GetEdgeeNodePublicIP(v) if err != nil { klog.ErrorS(err, "unable to get node public IP") return reconcile.Result{}, err diff --git a/pkg/yurtmanager/controller/raven/utils/utils.go b/pkg/yurtmanager/controller/raven/utils/utils.go index 8f3ae83e631..537ce7f38c7 100644 --- a/pkg/yurtmanager/controller/raven/utils/utils.go +++ b/pkg/yurtmanager/controller/raven/utils/utils.go @@ -40,7 +40,7 @@ const ( WorkingNamespace = "kube-system" RavenGlobalConfig = "raven-cfg" LabelCurrentGatewayEndpoints = "raven.openyurt.io/endpoints-name" - LabelLeptonSatellitePublicIP = "lepton.ai/provider-public-ip" + LabelLeptonSatellitePublicIP = "raven.openyurt.io/provider-public-ip" GatewayProxyInternalService = "x-raven-proxy-internal-svc" GatewayProxyServiceNamePrefix = "x-raven-proxy-svc" GatewayTunnelServiceNamePrefix = "x-raven-tunnel-svc" @@ -68,13 +68,13 @@ func GetNodeInternalIP(node corev1.Node) string { return ip } -func GetLeptonSatelliteNodePublicIP(node corev1.Node) (string, error) { +func GetEdgeeNodePublicIP(node corev1.Node) (string, error) { ip, ok := node.Labels[LabelLeptonSatellitePublicIP] if !ok { - return "", fmt.Errorf("Failed to get public ip, no label %s on node %s", LabelLeptonSatellitePublicIP, node.Name) + return "", fmt.Errorf("failed to get public ip, no label %s on node %s", LabelLeptonSatellitePublicIP, node.Name) } if net.ParseIP(ip) == nil { - return "", fmt.Errorf("Failed to get public ip, invalid public IP label %s, %s on node %s", LabelLeptonSatellitePublicIP, ip, node.Name) + return "", fmt.Errorf("failed to get public ip, invalid public IP label %s, %s on node %s", LabelLeptonSatellitePublicIP, ip, node.Name) } return ip, nil } From 1b802cd5c9e386a21e9fc2151873af4b993e0d63 Mon Sep 17 00:00:00 2001 From: suyuee Date: Mon, 16 Oct 2023 14:08:55 -0700 Subject: [PATCH 6/9] fix --- charts/yurt-manager/Chart.yaml | 4 ++-- charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml | 1 - pkg/apis/raven/v1beta1/gateway_types.go | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/charts/yurt-manager/Chart.yaml b/charts/yurt-manager/Chart.yaml index 7c795acc790..1f3c2bc1f2c 100644 --- a/charts/yurt-manager/Chart.yaml +++ b/charts/yurt-manager/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.3.4 +version: 1.3.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.3.4" +appVersion: "1.3.5" diff --git a/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml b/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml index aef214cb2d6..ea4da1c65ab 100644 --- a/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml +++ b/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml @@ -298,7 +298,6 @@ spec: required: - nodeName - privateIP - - publicIP - subnets type: object type: array diff --git a/pkg/apis/raven/v1beta1/gateway_types.go b/pkg/apis/raven/v1beta1/gateway_types.go index 43eef1e298b..3b87d590f0e 100644 --- a/pkg/apis/raven/v1beta1/gateway_types.go +++ b/pkg/apis/raven/v1beta1/gateway_types.go @@ -101,7 +101,7 @@ type NodeInfo struct { Subnets []string `json:"subnets"` // Node PublicIP: satellite nodes need to build vxlan connctions // using their public IPs; - PublicIP string `json:"publicIP"` + PublicIP string `json:"publicIP,omitempty"` } // GatewayStatus defines the observed state of Gateway From eb051b5d0a31cb28853a099817e1d9ec7292d0ee Mon Sep 17 00:00:00 2001 From: suyuee Date: Mon, 16 Oct 2023 14:49:34 -0700 Subject: [PATCH 7/9] tune down to info --- ...lockaffinities.apiregistration.k8s.io.yaml | 62 ------------------- .../gateway_pickup_controller.go | 3 +- 2 files changed, 1 insertion(+), 64 deletions(-) delete mode 100644 charts/yurt-manager/crds/apiextensions.k8s.io_v1_customresourcedefinition_blockaffinities.apiregistration.k8s.io.yaml diff --git a/charts/yurt-manager/crds/apiextensions.k8s.io_v1_customresourcedefinition_blockaffinities.apiregistration.k8s.io.yaml b/charts/yurt-manager/crds/apiextensions.k8s.io_v1_customresourcedefinition_blockaffinities.apiregistration.k8s.io.yaml deleted file mode 100644 index ab1b7eb8077..00000000000 --- a/charts/yurt-manager/crds/apiextensions.k8s.io_v1_customresourcedefinition_blockaffinities.apiregistration.k8s.io.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.7.0 - creationTimestamp: null - name: blockaffinities.apiregistration.k8s.io -spec: - group: apiregistration.k8s.io - names: - kind: BlockAffinity - listKind: BlockAffinityList - plural: blockaffinities - singular: blockaffinity - scope: Namespaced - versions: - - name: v3 - schema: - openAPIV3Schema: - description: BlockAffinity maintains a block affinity's state - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: Specification of the BlockAffinity. - properties: - cidr: - type: string - deleted: - description: Deleted indicates that this block affinity is being deleted. - This field is a string for compatibility with older releases that - mistakenly treat this field as a string. - type: string - node: - type: string - state: - type: string - required: - - cidr - - deleted - - node - - state - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go index c13d133a381..e37c3501d12 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go @@ -194,8 +194,7 @@ func (r *ReconcileGateway) Reconcile(ctx context.Context, req reconcile.Request) publicIP := "" publicIP, err = utils.GetEdgeeNodePublicIP(v) if err != nil { - klog.ErrorS(err, "unable to get node public IP") - return reconcile.Result{}, err + klog.InfoS("unable to get node public IP, expecting nodes to communicate via private IPs: " + err.Error()) } nodes = append(nodes, ravenv1beta1.NodeInfo{ NodeName: v.Name, From 5fdff4de415babf0e8c7950cbf470d551ae3daf5 Mon Sep 17 00:00:00 2001 From: suyuee Date: Mon, 16 Oct 2023 16:54:08 -0700 Subject: [PATCH 8/9] fix upstream breaking change --- charts/yurt-manager/templates/yurt-manager.yaml | 2 +- charts/yurt-manager/values.yaml | 1 + .../raven/gatewaypickup/gateway_pickup_controller.go | 7 +++++-- pkg/yurtmanager/controller/raven/utils/utils.go | 6 ++++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/charts/yurt-manager/templates/yurt-manager.yaml b/charts/yurt-manager/templates/yurt-manager.yaml index 7fe5908b353..3f84ca083a0 100644 --- a/charts/yurt-manager/templates/yurt-manager.yaml +++ b/charts/yurt-manager/templates/yurt-manager.yaml @@ -80,7 +80,7 @@ spec: command: - /usr/local/bin/yurt-manager image: {{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ .Values.image.pullPolicy }} name: yurt-manager ports: - containerPort: {{ .Values.ports.webhook }} diff --git a/charts/yurt-manager/values.yaml b/charts/yurt-manager/values.yaml index a2c5152f028..79cd0ec40bc 100644 --- a/charts/yurt-manager/values.yaml +++ b/charts/yurt-manager/values.yaml @@ -14,6 +14,7 @@ image: registry: openyurt repository: yurt-manager tag: v1.3.4 + pullPolicy: always ports: metrics: 10271 diff --git a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go index e37c3501d12..4cb4df7d2e3 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go @@ -177,9 +177,10 @@ func (r *ReconcileGateway) Reconcile(ctx context.Context, req reconcile.Request) err = fmt.Errorf("unable to list nodes: %s", err) return reconcile.Result{}, err } - klog.V(1).Info(Format("list gateway %d node %v", len(nodeList.Items), nodeList.Items)) + klog.V(1).Info(Format("list gateway %d", len(nodeList.Items))) // 1. try to elect an active endpoint if possible activeEp := r.electActiveEndpoint(nodeList, &gw) + klog.V(1).Infof("Elected %d endpoint for gateway %s: %v", len(activeEp), gw.Name, activeEp) r.recordEndpointEvent(&gw, gw.Status.ActiveEndpoints, activeEp) gw.Status.ActiveEndpoints = activeEp r.configEndpoints(ctx, &gw) @@ -255,10 +256,11 @@ func (r *ReconcileGateway) electActiveEndpoint(nodeList corev1.NodeList, gw *rav readyNodes[v.Name] = &v } } - klog.V(1).Infof(Format("Ready node has %d, node %v", len(readyNodes), readyNodes)) + klog.V(1).Infof(Format("Ready node has %d", len(readyNodes))) // init a endpoints slice enableProxy, enableTunnel := utils.CheckServer(context.TODO(), r.Client) eps := make([]*ravenv1beta1.Endpoint, 0) + klog.V(1).Info("enableProxy=%t, enableTunnel=%t", enableProxy, enableTunnel) if enableProxy { eps = append(eps, electEndpoints(gw, ravenv1beta1.Proxy, readyNodes)...) } @@ -270,6 +272,7 @@ func (r *ReconcileGateway) electActiveEndpoint(nodeList corev1.NodeList, gw *rav } func electEndpoints(gw *ravenv1beta1.Gateway, endpointType string, readyNodes map[string]*corev1.Node) []*ravenv1beta1.Endpoint { + klog.V(1).Infof("Electing endpoints for gateway %s with endpoint type %s", gw.Name, endpointType) eps := make([]*ravenv1beta1.Endpoint, 0) var replicas int switch endpointType { diff --git a/pkg/yurtmanager/controller/raven/utils/utils.go b/pkg/yurtmanager/controller/raven/utils/utils.go index 537ce7f38c7..01611809984 100644 --- a/pkg/yurtmanager/controller/raven/utils/utils.go +++ b/pkg/yurtmanager/controller/raven/utils/utils.go @@ -54,6 +54,8 @@ const ( VPNServerExposedPortKey = "tunnel-bind-addr" RavenEnableProxy = "enable-l7-proxy" RavenEnableTunnel = "enable-l3-tunnel" + DefaultEnableL7Proxy = false + DefaultEnableL3Tunnel = true ) // GetNodeInternalIP returns internal ip of the given `node`. @@ -91,8 +93,8 @@ func AddGatewayToWorkQueue(gwName string, func CheckServer(ctx context.Context, client client.Client) (enableProxy, enableTunnel bool) { var cm corev1.ConfigMap - enableTunnel = false - enableProxy = false + enableTunnel = DefaultEnableL3Tunnel + enableProxy = DefaultEnableL7Proxy err := client.Get(ctx, types.NamespacedName{Namespace: WorkingNamespace, Name: RavenGlobalConfig}, &cm) if err != nil { return enableProxy, enableTunnel From fd225fa0f2543104d5b0e26817f30bdba1acf383 Mon Sep 17 00:00:00 2001 From: suyuee Date: Sun, 22 Oct 2023 17:04:42 -0700 Subject: [PATCH 9/9] fix --- .../yurt-manager/crds/raven.openyurt.io_gateways.yaml | 3 +++ charts/yurt-manager/values.yaml | 2 +- pkg/apis/raven/v1alpha1/gateway_conversion.go | 2 ++ pkg/apis/raven/v1alpha1/gateway_types.go | 3 +++ .../raven/gatewaypickup/gateway_pickup_controller.go | 2 +- pkg/yurtmanager/controller/raven/utils/utils.go | 10 +++++----- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml b/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml index ea4da1c65ab..0578f269616 100644 --- a/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml +++ b/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml @@ -123,6 +123,9 @@ spec: type: string privateIP: type: string + publicIP: + description: 'Node PublicIP: satellite nodes need to build vxlan connctions using their public IPs;' + type: string subnets: items: type: string diff --git a/charts/yurt-manager/values.yaml b/charts/yurt-manager/values.yaml index 79cd0ec40bc..a5da8d8aee3 100644 --- a/charts/yurt-manager/values.yaml +++ b/charts/yurt-manager/values.yaml @@ -14,7 +14,7 @@ image: registry: openyurt repository: yurt-manager tag: v1.3.4 - pullPolicy: always + pullPolicy: Always ports: metrics: 10271 diff --git a/pkg/apis/raven/v1alpha1/gateway_conversion.go b/pkg/apis/raven/v1alpha1/gateway_conversion.go index 6192bfc4a21..388ff1df141 100644 --- a/pkg/apis/raven/v1alpha1/gateway_conversion.go +++ b/pkg/apis/raven/v1alpha1/gateway_conversion.go @@ -62,6 +62,7 @@ func (src *Gateway) ConvertTo(dstRaw conversion.Hub) error { NodeName: node.NodeName, PrivateIP: node.PrivateIP, Subnets: node.Subnets, + PublicIP: node.PublicIP, }) } if src.Status.ActiveEndpoint != nil { @@ -102,6 +103,7 @@ func (dst *Gateway) ConvertFrom(srcRaw conversion.Hub) error { NodeName: node.NodeName, PrivateIP: node.PrivateIP, Subnets: node.Subnets, + PublicIP: node.PublicIP, }) } if src.Status.ActiveEndpoints == nil { diff --git a/pkg/apis/raven/v1alpha1/gateway_types.go b/pkg/apis/raven/v1alpha1/gateway_types.go index 551254e8bb0..9b6c4d717e6 100644 --- a/pkg/apis/raven/v1alpha1/gateway_types.go +++ b/pkg/apis/raven/v1alpha1/gateway_types.go @@ -70,6 +70,9 @@ type NodeInfo struct { NodeName string `json:"nodeName"` PrivateIP string `json:"privateIP"` Subnets []string `json:"subnets"` + // Node PublicIP: satellite nodes need to build vxlan connctions + // using their public IPs; + PublicIP string `json:"publicIP,omitempty"` } // GatewayStatus defines the observed state of Gateway diff --git a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go index 4cb4df7d2e3..d685572dc72 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go @@ -193,7 +193,7 @@ func (r *ReconcileGateway) Reconcile(ctx context.Context, req reconcile.Request) return reconcile.Result{}, err } publicIP := "" - publicIP, err = utils.GetEdgeeNodePublicIP(v) + publicIP, err = utils.GetEdgeNodePublicIP(&v) if err != nil { klog.InfoS("unable to get node public IP, expecting nodes to communicate via private IPs: " + err.Error()) } diff --git a/pkg/yurtmanager/controller/raven/utils/utils.go b/pkg/yurtmanager/controller/raven/utils/utils.go index 01611809984..b44b36c2afa 100644 --- a/pkg/yurtmanager/controller/raven/utils/utils.go +++ b/pkg/yurtmanager/controller/raven/utils/utils.go @@ -40,7 +40,7 @@ const ( WorkingNamespace = "kube-system" RavenGlobalConfig = "raven-cfg" LabelCurrentGatewayEndpoints = "raven.openyurt.io/endpoints-name" - LabelLeptonSatellitePublicIP = "raven.openyurt.io/provider-public-ip" + LabelNodeProviderPublicIP = "raven.openyurt.io/provider-public-ip" GatewayProxyInternalService = "x-raven-proxy-internal-svc" GatewayProxyServiceNamePrefix = "x-raven-proxy-svc" GatewayTunnelServiceNamePrefix = "x-raven-tunnel-svc" @@ -70,13 +70,13 @@ func GetNodeInternalIP(node corev1.Node) string { return ip } -func GetEdgeeNodePublicIP(node corev1.Node) (string, error) { - ip, ok := node.Labels[LabelLeptonSatellitePublicIP] +func GetEdgeNodePublicIP(node *corev1.Node) (string, error) { + ip, ok := node.Labels[LabelNodeProviderPublicIP] if !ok { - return "", fmt.Errorf("failed to get public ip, no label %s on node %s", LabelLeptonSatellitePublicIP, node.Name) + return "", fmt.Errorf("failed to get public ip, no label %s on node %s", LabelNodeProviderPublicIP, node.Name) } if net.ParseIP(ip) == nil { - return "", fmt.Errorf("failed to get public ip, invalid public IP label %s, %s on node %s", LabelLeptonSatellitePublicIP, ip, node.Name) + return "", fmt.Errorf("failed to get public ip, invalid public IP label %s, %s on node %s", LabelNodeProviderPublicIP, ip, node.Name) } return ip, nil }