Skip to content

Commit

Permalink
e2e for vip dual stack with security group (#3630)
Browse files Browse the repository at this point in the history 
Signed-off-by: zcq98 <[email protected]>
Co-authored-by: zcq98 <[email protected]>
  • Loading branch information
@zhaocongqi @zbb88888
2 people authored and zbb88888 committed Jan 17, 2024
1 parent 1005dd9 commit 1814b80
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 105 deletions.
1 change: 0 additions & 1 deletion pkg/util/net.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,6 @@ func SplitStringIP(ipStr string) (string, string) {
v4IP = ipStr
case kubeovnv1.ProtocolIPv6:
v6IP = ipStr

}

return v4IP, v6IP
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/kube-ovn/pod/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ var _ = framework.Describe("[group:pod]", func() {
})

framework.ConformanceIt("should support configuring routes via pod annotation", func() {
f.SkipVersionPriorTo(1, 12, "This feature was introduced inn v1.12")
f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12")

ginkgo.By("Generating routes")
routes := make([]request.Route, 0, 4)
Expand Down
270 changes: 167 additions & 103 deletions test/e2e/vip/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,86 @@ func makeOvnVip(namespaceName, name, subnet, v4ip, v6ip, vipType string) *apiv1.
return framework.MakeVip(namespaceName, name, subnet, v4ip, v6ip, vipType)
}

func MakeSecurityGroup(name string, allowSameGroupTraffic bool, ingressRules, egressRules []*apiv1.SgRule) *apiv1.SecurityGroup {
func makeSecurityGroup(name string, allowSameGroupTraffic bool, ingressRules, egressRules []*apiv1.SgRule) *apiv1.SecurityGroup {
return framework.MakeSecurityGroup(name, allowSameGroupTraffic, ingressRules, egressRules)
}

func testConnectivity(ip, namespaceName, srcPod, dstPod string, f *framework.Framework) {
// other pods can communicate with the allow address pair pod through vip
var addIP, delIP, command string
switch util.CheckProtocol(ip) {
case apiv1.ProtocolIPv4:
addIP = fmt.Sprintf("ip addr add %s/24 dev eth0", ip)
delIP = fmt.Sprintf("ip addr del %s/24 dev eth0", ip)
command = fmt.Sprintf("ping -W 1 -c 1 %s", ip)
case apiv1.ProtocolIPv6:
addIP = fmt.Sprintf("ip addr add %s/96 dev eth0", ip)
delIP = fmt.Sprintf("ip addr del %s/96 dev eth0", ip)
command = fmt.Sprintf("ping6 -c 1 %s", ip)
}
// check srcPod ping dstPod through vip
stdout, stderr, err := framework.ExecShellInPod(context.Background(), f, namespaceName, dstPod, addIP)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", addIP, err, stderr, stdout)
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, srcPod, command)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout)
framework.ExpectNoError(err)
// srcPod can not ping dstPod vip when ip is deleted
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, dstPod, delIP)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", delIP, err, stderr, stdout)
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, srcPod, command)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout)
framework.ExpectError(err)
// check dstPod ping srcPod through vip
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, srcPod, addIP)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", addIP, err, stderr, stdout)
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, dstPod, command)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout)
framework.ExpectNoError(err)
// dstPod can not ping srcPod vip when ip is deleted
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, srcPod, delIP)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", delIP, err, stderr, stdout)
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, dstPod, command)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout)
framework.ExpectError(err)
}

func testVipWithSG(ip, namespaceName, allowPod, denyPod, aapPod, securityGroupName string, f *framework.Framework) {
// check if security group working
var sgCheck, conditions string
switch util.CheckProtocol(ip) {
case apiv1.ProtocolIPv4:
sgCheck = fmt.Sprintf("ping -W 1 -c 1 %s", ip)
conditions = fmt.Sprintf("name=ovn.sg.%s.associated.v4", strings.ReplaceAll(securityGroupName, "-", "."))
case apiv1.ProtocolIPv6:
sgCheck = fmt.Sprintf("ping6 -c 1 %s", ip)
conditions = fmt.Sprintf("name=ovn.sg.%s.associated.v6", strings.ReplaceAll(securityGroupName, "-", "."))
}
// allowPod can ping aapPod with security group
stdout, stderr, err := framework.ExecShellInPod(context.Background(), f, namespaceName, allowPod, sgCheck)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", sgCheck, err, stderr, stdout)
framework.ExpectNoError(err)
// denyPod can not ping aapPod with security group
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, denyPod, sgCheck)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", sgCheck, err, stderr, stdout)
framework.ExpectError(err)

ginkgo.By("Checking ovn address_set and lsp port_security")
// address_set should have allow address pair ip
execCmd := "kubectl ko nbctl --format=list --data=bare --no-heading --columns=addresses find address_set " + conditions
output, err := exec.Command("bash", "-c", execCmd).CombinedOutput()
addressSet := strings.Split(strings.ReplaceAll(string(output), "\n", ""), " ")
framework.ExpectNoError(err)
framework.ExpectContainElement(addressSet, ip)
// port_security should have allow address pair IP
execCmd = fmt.Sprintf("kubectl ko nbctl --format=list --data=bare --no-heading --columns=port_security find logical-switch-port name=%s.%s", aapPod, namespaceName)
output, err = exec.Command("bash", "-c", execCmd).CombinedOutput()
portSecurity := strings.Split(strings.ReplaceAll(string(output), "\n", ""), " ")
framework.ExpectNoError(err)
framework.ExpectContainElement(portSecurity, ip)
// TODO: Checking allow address pair connectivity with security group
// AAP does not work fine with security group in kind test env for now
}

var _ = framework.Describe("[group:vip]", func() {
f := framework.NewDefaultFramework("vip")

Expand Down Expand Up @@ -121,78 +197,65 @@ var _ = framework.Describe("[group:vip]", func() {

framework.ConformanceIt("Test vip", func() {
ginkgo.By("1. Test allowed address pair vip")
annotations := map[string]string{util.AAPsAnnotation: vip1Name}
cmd := []string{"sh", "-c", "sleep infinity"}
ginkgo.By("Creating pod1 support allowed address pair using " + vip1Name)
aapPod1 := framework.MakeNetAdminPod(namespaceName, aapPodName1, nil, annotations, image, cmd, nil)
aapPod1 = podClient.CreateSync(aapPod1)
ginkgo.By("Creating pod2 support allowed address pair using " + vip1Name)
aapPod2 := framework.MakeNetAdminPod(namespaceName, aapPodName2, nil, annotations, image, cmd, nil)
_ = podClient.CreateSync(aapPod2)
// create vip1 and vip2, should have different ip and mac
ginkgo.By("Creating allowed address pair vip, should have different ip and mac")
ginkgo.By("Creating allowed address pair vip " + vip1Name)
vip1 := makeOvnVip(namespaceName, vip1Name, subnetName, "", "", "")
vip1 = vipClient.CreateSync(vip1)
ginkgo.By("Creating allowed address pair vip " + vip2Name)
vip2 := makeOvnVip(namespaceName, vip2Name, subnetName, "", "", "")
vip2 = vipClient.CreateSync(vip2)
// arp proxy vip only used in switch lb rule, the lb vip use the subnet gw mac to use lb nat flow
virtualIP1 := util.GetStringIP(vip1.Status.V4ip, vip1.Status.V6ip)
virtualIP2 := util.GetStringIP(vip2.Status.V4ip, vip2.Status.V6ip)
framework.ExpectNotEqual(virtualIP1, virtualIP2)
framework.ExpectNotEqual(vip1.Status.Mac, vip2.Status.Mac)
if vip1.Status.V4ip != "" {
framework.ExpectNotEqual(vip1.Status.V4ip, vip2.Status.V4ip)
// logical switch port with type virtual should be created
conditions := fmt.Sprintf("type=virtual name=%s options:virtual-ip=%s ", vip1Name, vip1.Status.V4ip)
execCmd := "kubectl ko nbctl --format=list --data=bare --no-heading --columns=options find logical-switch-port " + conditions
output, err := exec.Command("bash", "-c", execCmd).CombinedOutput()
framework.ExpectNoError(err)
framework.ExpectNotEmpty(strings.TrimSpace(string(output)))
// virtual parents should be set correctlly
pairs := strings.Split(string(output), " ")
options := make(map[string]string)
for _, pair := range pairs {
keyValue := strings.Split(pair, "=")
if len(keyValue) == 2 {
options[keyValue[0]] = strings.ReplaceAll(keyValue[1], "\n", "")
}

annotations := map[string]string{util.AAPsAnnotation: vip1Name}
cmd := []string{"sh", "-c", "sleep infinity"}
ginkgo.By("Creating pod1 support allowed address pair using " + vip1Name)
aapPod1 := framework.MakeNetAdminPod(namespaceName, aapPodName1, nil, annotations, image, cmd, nil)
aapPod1 = podClient.CreateSync(aapPod1)
ginkgo.By("Creating pod2 support allowed address pair using " + vip1Name)
aapPod2 := framework.MakeNetAdminPod(namespaceName, aapPodName2, nil, annotations, image, cmd, nil)
_ = podClient.CreateSync(aapPod2)
// logical switch port with type virtual should be created
conditions := fmt.Sprintf("type=virtual name=%s options:virtual-ip=\\\"%s\\\" ", vip1Name, virtualIP1)
execCmd := "kubectl ko nbctl --format=list --data=bare --no-heading --columns=options find logical-switch-port " + conditions
framework.Logf("exec cmd %s", execCmd)
output, err := exec.Command("bash", "-c", execCmd).CombinedOutput()
framework.ExpectNoError(err)
framework.ExpectNotEmpty(strings.TrimSpace(string(output)))
// virtual parents should be set correctlly
pairs := strings.Split(string(output), " ")
options := make(map[string]string)
for _, pair := range pairs {
keyValue := strings.Split(pair, "=")
if len(keyValue) == 2 {
options[keyValue[0]] = strings.ReplaceAll(keyValue[1], "\n", "")
}
virtualParents := strings.Split(options["virtual-parents"], ",")
sort.Strings(virtualParents)
expectVirtualParents := []string{fmt.Sprintf("%s.%s", aapPodName1, namespaceName), fmt.Sprintf("%s.%s", aapPodName2, namespaceName)}
sort.Strings(expectVirtualParents)
framework.ExpectEqual(expectVirtualParents, virtualParents)
// other pods can communicate with the aap pod through vip
ginkgo.By("Test pod ping aap address " + vip1.Status.V4ip)
addIP := fmt.Sprintf("ip addr add %s/24 dev eth0", vip1.Status.V4ip)
delIP := fmt.Sprintf("ip addr del %s/24 dev eth0", vip1.Status.V4ip)
command := fmt.Sprintf("ping -W 1 -c 1 %s", vip1.Status.V4ip)
// check aapPod2 ping aapPod1 through vip
stdout, stderr, err := framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName1, addIP)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", addIP, err, stderr, stdout)
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName2, command)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout)
framework.ExpectNoError(err)
// aapPod2 can not ping aapPod1 vip when ip is deleted
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName1, delIP)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", delIP, err, stderr, stdout)
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName2, command)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout)
framework.ExpectError(err)
// check aapPod1 ping aapPod2 through vip
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName2, addIP)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", addIP, err, stderr, stdout)
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName1, command)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout)
framework.ExpectNoError(err)
// aapPod1 can not ping aapPod2 vip when ip is deleted
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName2, delIP)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", delIP, err, stderr, stdout)
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName1, command)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout)
framework.ExpectError(err)
ginkgo.By("Creating security group " + securityGroupName)
gatewayV4 := aapPod1.Annotations[util.GatewayAnnotation]
allowAddress := aapPod1.Annotations[util.IPAddressAnnotation]
rules := make([]*apiv1.SgRule, 0, 2)
}
virtualParents := strings.Split(options["virtual-parents"], ",")
sort.Strings(virtualParents)
expectVirtualParents := []string{fmt.Sprintf("%s.%s", aapPodName1, namespaceName), fmt.Sprintf("%s.%s", aapPodName2, namespaceName)}
sort.Strings(expectVirtualParents)
framework.ExpectEqual(expectVirtualParents, virtualParents)

ginkgo.By("Test allow address pair connectivity")
if f.HasIPv4() {
ginkgo.By("Test pod ping allow address pair " + vip1.Status.V4ip)
testConnectivity(vip1.Status.V4ip, namespaceName, aapPodName2, aapPodName1, f)
}
if f.HasIPv6() {
ginkgo.By("Test pod ping allow address pair " + vip1.Status.V6ip)
testConnectivity(vip1.Status.V6ip, namespaceName, aapPodName2, aapPodName1, f)
}

ginkgo.By("3. Test vip with security group")
ginkgo.By("Creating security group " + securityGroupName)
gatewayV4, gatewayV6 := util.SplitStringIP(aapPod1.Annotations[util.GatewayAnnotation])
allowAddressV4, allowAddressV6 := util.SplitStringIP(aapPod1.Annotations[util.IPAddressAnnotation])
rules := make([]*apiv1.SgRule, 0)
if f.HasIPv4() {
// gateway should be added for pinger
rules = append(rules, &apiv1.SgRule{
IPVersion: "ipv4",
Expand All @@ -202,54 +265,55 @@ var _ = framework.Describe("[group:vip]", func() {
RemoteAddress: gatewayV4,
Policy: apiv1.PolicyAllow,
})
// aapPod1 should be allowed by aapPod3 for security group aap test
// aapPod1 should be allowed by aapPod3 for security group allow address pair test
rules = append(rules, &apiv1.SgRule{
IPVersion: "ipv4",
Protocol: apiv1.ProtocolALL,
Priority: 1,
RemoteType: apiv1.SgRemoteTypeAddress,
RemoteAddress: allowAddress,
RemoteAddress: allowAddressV4,
Policy: apiv1.PolicyAllow,
})
sg := MakeSecurityGroup(securityGroupName, true, rules, rules)
_ = securityGroupClient.CreateSync(sg)
ginkgo.By("Creating pod3 support allowed address pair with security group")
annotations[util.PortSecurityAnnotation] = "true"
annotations[fmt.Sprintf(util.SecurityGroupAnnotationTemplate, "ovn")] = securityGroupName
aapPod3 := framework.MakeNetAdminPod(namespaceName, aapPodName3, nil, annotations, image, cmd, nil)
aapPod3 = podClient.CreateSync(aapPod3)
// check if security group working
sgCheck := fmt.Sprintf("ping -W 1 -c 1 %s", aapPod3.Annotations[util.IPAddressAnnotation])
// aapPod1 can ping aapPod3 with security group
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName1, sgCheck)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", sgCheck, err, stderr, stdout)
framework.ExpectNoError(err)
// aapPod3 can not ping aapPod3 with security group
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName2, sgCheck)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", sgCheck, err, stderr, stdout)
framework.ExpectError(err)
ginkgo.By("Checking ovn address_set and lsp port_security")
// address_set should have aap IP
conditions = fmt.Sprintf("name=ovn.sg.%s.associated.v4", strings.ReplaceAll(securityGroupName, "-", "."))
execCmd = "kubectl ko nbctl --format=list --data=bare --no-heading --columns=addresses find address_set " + conditions
output, err = exec.Command("bash", "-c", execCmd).CombinedOutput()
addressSet := strings.Split(strings.ReplaceAll(string(output), "\n", ""), " ")
framework.ExpectNoError(err)
framework.ExpectContainElement(addressSet, vip1.Status.V4ip)
// port_security should have aap IP
conditions = fmt.Sprintf("name=%s.%s", aapPodName3, namespaceName)
execCmd = "kubectl ko nbctl --format=list --data=bare --no-heading --columns=port_security find logical-switch-port " + conditions
output, err = exec.Command("bash", "-c", execCmd).CombinedOutput()
portSecurity := strings.Split(strings.ReplaceAll(string(output), "\n", ""), " ")
framework.ExpectNoError(err)
framework.ExpectContainElement(portSecurity, vip1.Status.V4ip)
// TODO: Checking allow address pair connectivity with security group
// AAP works fine with security group in kind but not working in e2e
} else {
framework.ExpectNotEqual(vip1.Status.V6ip, vip2.Status.V6ip)
}
if f.HasIPv6() {
// gateway should be added for pinger
rules = append(rules, &apiv1.SgRule{
IPVersion: "ipv6",
Protocol: apiv1.ProtocolALL,
Priority: 1,
RemoteType: apiv1.SgRemoteTypeAddress,
RemoteAddress: gatewayV6,
Policy: apiv1.PolicyAllow,
})
// aapPod1 should be allowed by aapPod3 for security group allow address pair test
rules = append(rules, &apiv1.SgRule{
IPVersion: "ipv6",
Protocol: apiv1.ProtocolALL,
Priority: 1,
RemoteType: apiv1.SgRemoteTypeAddress,
RemoteAddress: allowAddressV6,
Policy: apiv1.PolicyAllow,
})
}
sg := makeSecurityGroup(securityGroupName, true, rules, rules)
_ = securityGroupClient.CreateSync(sg)

ginkgo.By("Creating pod3 support allowed address pair with security group")
annotations[util.PortSecurityAnnotation] = "true"
annotations[fmt.Sprintf(util.SecurityGroupAnnotationTemplate, "ovn")] = securityGroupName
aapPod3 := framework.MakeNetAdminPod(namespaceName, aapPodName3, nil, annotations, image, cmd, nil)
aapPod3 = podClient.CreateSync(aapPod3)
v4ip, v6ip := util.SplitStringIP(aapPod3.Annotations[util.IPAddressAnnotation])
if f.HasIPv4() {
ginkgo.By("Test allow address pair with security group for ipv4")
testVipWithSG(v4ip, namespaceName, aapPodName1, aapPodName2, aapPodName3, securityGroupName, f)
}
if f.HasIPv6() {
ginkgo.By("Test allow address pair with security group for ipv6")
testVipWithSG(v6ip, namespaceName, aapPodName1, aapPodName2, aapPodName3, securityGroupName, f)
}

ginkgo.By("2. Test switch lb vip")
ginkgo.By("3. Test switch lb vip")
ginkgo.By("Creating two arp proxy vips, should have the same mac which is from gw subnet mac")
ginkgo.By("Creating arp proxy switch lb vip " + switchLbVip1Name)
switchLbVip1 := makeOvnVip(namespaceName, switchLbVip1Name, subnetName, "", "", util.SwitchLBRuleVip)
Expand Down

0 comments on commit 1814b80

Please sign in to comment.