[Bug]: bgp/test_bgp_suppress_fib.py::test_bgp_route_with_suppres failed at internal PR test

[StormLiangMS]

Issue Description

https://elastictest.org/scheduler/testplan/66da09e1f90c6cd947f4e495?testcase=bgp%2Ftest_bgp_suppress_fib.py%7C%7C%7C2&type=console&leftSideViewMode=detail&prop=status&order=ascending&searchTestCase=

failed at internal PR test and block the internal sync for weeks.

Results you see

bgp/test_bgp_suppress_fib.py::test_bgp_route_with_suppress[default]
-------------------------------- live log call ---------------------------------
21:47:17 init.pytest_runtest_call L0040 ERROR | Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/_pytest/python.py", line 1788, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
File "/usr/local/lib/python3.8/dist-packages/pluggy/_hooks.py", line 513, in call
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
File "/usr/local/lib/python3.8/dist-packages/pluggy/_manager.py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
File "/usr/local/lib/python3.8/dist-packages/pluggy/_callers.py", line 139, in _multicall
raise exception.with_traceback(exception.traceback)
File "/usr/local/lib/python3.8/dist-packages/pluggy/_callers.py", line 103, in _multicall
res = hook_impl.function(args)
File "/usr/local/lib/python3.8/dist-packages/_pytest/python.py", line 194, in pytest_pyfunc_call
result = testfunction(**testargs)
File "/var/src/sonic-mgmt/tests/bgp/test_bgp_suppress_fib.py", line 766, in test_bgp_route_with_suppress
validate_traffic(ptfadapter, [traffic_data_ipv4_drop, traffic_data_ipv6_drop], router_mac,
File "/var/src/sonic-mgmt/tests/bgp/test_bgp_suppress_fib.py", line 568, in validate_traffic
send_and_verify_packet(ptfadapter, pkt_list, exp_pkt_list, tx_port, recv_port, exp_res_list, ip_ver_list)
File "/var/src/sonic-mgmt/tests/bgp/test_bgp_suppress_fib.py", line 389, in send_and_verify_packet
testutils.verify_no_packet(ptfadapter, pkt=exp_pkt, port_id=rx_port, timeout=TRAFFIC_WAIT_TIME)
File "/usr/local/lib/python3.8/dist-packages/ptf/testutils.py", line 3270, in verify_no_packet
test.fail(
File "/usr/lib/python3.8/unittest/case.py", line 753, in fail
raise self.failureException(msg)
AssertionError: Received packet that we expected not to receive on device 0, port 27.
========== RECEIVED ==========
0000 26 C3 34 15 36 F7 22 2F C5 5B BB B0 08 00 45 00 &.4.6."/.[....E.
0010 00 56 00 01 00 00 3F 00 5E FC C0 A8 00 02 5B 00 .V....?.^.....[.
0020 01 01 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D ................
0030 0E 0F 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D ................
0040 1E 1F 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D .. !"#$%&'()+,-
0050 2E 2F 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D ./0123456789:;<=
0060 3E 3F 40 41 >?@A

FAILED [ 14%]Thursday 05 September 2024 21:49:24 +0000 (0:02:00.718) 0:09:14.909 ****

=================================== FAILURES ===================================
____________________ test_bgp_route_with_suppress[default] _____________________

duthost =
tbinfo = {'auto_recover': 'False', 'comment': 'Tests virtual switch vm', 'conf-name': 'vms-kvm-t1-lag', 'duts': ['vlab-03'], ...}
nbrhosts = {'ARISTA01T0': , 'ARISTA01T2': , 'ARISTA02T0': , 'ARISTA03T0': , ...}
ptfadapter = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
localhost = <tests.common.devices.local.Localhost object at 0x7fa27e4ca520>
restore_bgp_suppress_fib = None
prepare_param = ('22:2f:c5:5b:bb:b0', {'deployment_id': '1', 'dhcp_servers': [], 'dhcpv6_servers': [], 'forced_mgmt_routes': ['172.17.0.1/24'], ...}, '10.250.0.106', (5019,), (6011,), [{4: 27, 6: 19}])
vrf_type = 'default', continuous_boot_times = 1
generate_route_and_traffic_data = {'function': [['91.0.1.0/24', '91.0.2.0/24'], ['1000:1001::/64', '1000:1001:0:1::/64'], [('91.0.1.1', 'FORWARD'), ('91...1001:0:2::1', 'DROP'), ('1000:1001:0:3::1', 'DROP'), ('1000:1001:0:4::1', 'DROP'), ('1000:1001:0:5::1', 'DROP'), ...]]}
request = <FixtureRequest for <Function test_bgp_route_with_suppress[default]>>

@pytest.mark.parametrize("vrf_type", VRF_TYPES)
def test_bgp_route_with_suppress(duthost, tbinfo, nbrhosts, ptfadapter, localhost, restore_bgp_suppress_fib,
                                 prepare_param, vrf_type, continuous_boot_times, generate_route_and_traffic_data,
                                 request):
    try:
        if vrf_type == USER_DEFINED_VRF:
            with allure.step("Configure user defined vrf"):
                setup_vrf(duthost, nbrhosts, tbinfo)

        with allure.step("Prepare needed parameters"):
            router_mac, mg_facts, ptf_ip, exabgp_port_list, exabgp_port_list_v6, recv_port_list = prepare_param

        with allure.step("Get route and traffic data"):
            ipv4_route_list, ipv6_route_list, traffic_data_ipv4_forward, traffic_data_ipv6_forward, \
                traffic_data_ipv4_drop, traffic_data_ipv6_drop = generate_route_and_traffic_data[FUNCTION]

        with allure.step("Config bgp suppress-fib-pending function"):
            config_bgp_suppress_fib(duthost)

        with allure.step("Save configuration"):
            logger.info("Save configuration")
            duthost.shell('sudo config save -y')

        for continous_boot_index in range(continuous_boot_times):
            if continuous_boot_times > 1:
                logger.info("======== Continuous boot needed - this is the {} time boot test ========".
                            format(continous_boot_index+1))

            with allure.step("Do reload"):
                param_reboot(request, duthost, localhost)

            for exabgp_port, exabgp_port_v6, recv_port in zip(exabgp_port_list, exabgp_port_list_v6, recv_port_list):
                try:
                    with allure.step("Suspend orchagent process to simulate a route install delay"):
                        operate_orchagent(duthost)

                    with allure.step(f"Announce BGP ipv4 and ipv6 routes to DUT from T0 VM by ExaBGP - "
                                     f"v4: {exabgp_port} v6: {exabgp_port_v6}"):
                        announce_ipv4_ipv6_routes(ptf_ip, ipv4_route_list, exabgp_port, ipv6_route_list, exabgp_port_v6)

                    with allure.step("Validate announced BGP ipv4 and ipv6 routes are in {} state".format(QUEUED)):
                        validate_route_states(duthost, ipv4_route_list, ipv6_route_list, vrf_type)

                    with allure.step("Validate BGP ipv4 and ipv6 routes are not announced to T2 VM peer"):
                        validate_route_propagate(duthost, nbrhosts, tbinfo, ipv4_route_list, ipv6_route_list, vrf_type,
                                                 exist=False)

                    with allure.step("Validate traffic could not be forwarded to T0 VM"):
                        ptf_interfaces = get_t2_ptf_intfs(mg_facts)

                      validate_traffic(ptfadapter, [traffic_data_ipv4_drop, traffic_data_ipv6_drop], router_mac,

                                         ptf_interfaces, recv_port)

continous_boot_index = 0
continuous_boot_times = 1
duthost =
exabgp_port = 5019
exabgp_port_list = (5019,)
exabgp_port_list_v6 = (6011,)
exabgp_port_v6 = 6011
generate_route_and_traffic_data = {'function': [['91.0.1.0/24', '91.0.2.0/24'], ['1000:1001::/64', '1000:1001:0:1::/64'], [('91.0.1.1', 'FORWARD'), ('91...1001:0:2::1', 'DROP'), ('1000:1001:0:3::1', 'DROP'), ('1000:1001:0:4::1', 'DROP'), ('1000:1001:0:5::1', 'DROP'), ...]]}
ipv4_route_list = ['91.0.1.0/24', '91.0.2.0/24']
ipv6_route_list = ['1000:1001::/64', '1000:1001:0:1::/64']
localhost = <tests.common.devices.local.Localhost object at 0x7fa27e4ca520>
mg_facts = {'deployment_id': '1', 'dhcp_servers': [], 'dhcpv6_servers': [], 'forced_mgmt_routes': ['172.17.0.1/24'], ...}
nbrhosts = {'ARISTA01T0': , 'ARISTA01T2': , 'ARISTA02T0': , 'ARISTA03T0': , ...}
prepare_param = ('22:2f:c5:5b:bb:b0', {'deployment_id': '1', 'dhcp_servers': [], 'dhcpv6_servers': [], 'forced_mgmt_routes': ['172.17.0.1/24'], ...}, '10.250.0.106', (5019,), (6011,), [{4: 27, 6: 19}])
ptf_interfaces = [0, 1, 2, 3, 4, 5, ...]
ptf_ip = '10.250.0.106'
ptfadapter = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
recv_port = {4: 27, 6: 19}
recv_port_list = [{4: 27, 6: 19}]
request = <FixtureRequest for <Function test_bgp_route_with_suppress[default]>>
restore_bgp_suppress_fib = None
router_mac = '22:2f:c5:5b:bb:b0'
tbinfo = {'auto_recover': 'False', 'comment': 'Tests virtual switch vm', 'conf-name': 'vms-kvm-t1-lag', 'duts': ['vlab-03'], ...}
traffic_data_ipv4_drop = [('91.0.1.1', 'DROP'), ('91.0.2.1', 'DROP')]
traffic_data_ipv4_forward = [('91.0.1.1', 'FORWARD'), ('91.0.2.1', 'FORWARD')]
traffic_data_ipv6_drop = [('1000:1001::1', 'DROP'), ('1000:1001:0:1::1', 'DROP')]
traffic_data_ipv6_forward = [('1000:1001::1', 'FORWARD'), ('1000:1001:0:1::1', 'FORWARD')]
vrf_type = 'default'

bgp/test_bgp_suppress_fib.py:766:

---

bgp/test_bgp_suppress_fib.py:568: in validate_traffic
send_and_verify_packet(ptfadapter, pkt_list, exp_pkt_list, tx_port, recv_port, exp_res_list, ip_ver_list)
exp_pkt_list = [<ptf.mask.Mask object at 0x7fa27e4e5370>, <ptf.mask.Mask object at 0x7fa27c1bcd60>]
exp_res_list = ['DROP', 'DROP']
ip_ver_list = [4, 4]
loop_back = False
pkt_list = [<Ether dst=22:2f:c5:5b:bb:b0 src=00:06:07:08:09:0a type=IPv4 |<IP ihl=None tos=0x0 id=1 ttl=64 proto=hopopt src=192...x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&'()+,-./0123456789:;<=>?@A' |>>>]
ptf_interfaces = [0, 1, 2, 3, 4, 5, ...]
ptfadapter = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
recv_port = {4: 27, 6: 19}
router_mac = '22:2f:c5:5b:bb:b0'
traffic_data = [('91.0.1.1', 'DROP'), ('91.0.2.1', 'DROP')]
traffic_data_list = [[('91.0.1.1', 'DROP'), ('91.0.2.1', 'DROP')], [('1000:1001::1', 'DROP'), ('1000:1001:0:1::1', 'DROP')]]
tx_port = 12
bgp/test_bgp_suppress_fib.py:389: in send_and_verify_packet
testutils.verify_no_packet(ptfadapter, pkt=exp_pkt, port_id=rx_port, timeout=TRAFFIC_WAIT_TIME)
exp_action = 'DROP'
exp_action_list = ['DROP', 'DROP']
exp_pkt = <ptf.mask.Mask object at 0x7fa27e4e5370>
exp_pkt_list = [<ptf.mask.Mask object at 0x7fa27e4e5370>, <ptf.mask.Mask object at 0x7fa27c1bcd60>]
ip_ver = 4
ip_ver_list = [4, 4]
DEBUG:tests.common.devices.base:/var/src/sonic-mgmt/tests/common/devices/sonic.py::get_up_time#1058: [vlab-03] AnsibleModule::command, args=["uptime -s"], kwargs={}
DEBUG:tests.common.devices.base:/var/src/sonic-mgmt/tests/common/devices/sonic.py::get_up_time#1058: [vlab-03] AnsibleModule::command Result => {"changed": true, "stdout": "2024-09-05 19:54:08", "stderr": "", "rc": 0, "cmd": ["uptime", "-s"], "start": "2024-09-05 21:49:25.063365", "end": "2024-09-05 21:49:25.072481", "delta": "0:00:00.009116", "msg": "", "invocation": {"module_args": {"_raw_params": "uptime -s", "_uses_shell": false, "warn": false, "stdin_add_newline": true, "strip_empty_ends": true, "argv": null, "chdir": null, "executable": null, "creates": null, "removes": null, "stdin": null}}, "stdout_lines": ["2024-09-05 19:54:08"], "stderr_lines": [], "_ansible_no_log": null, "failed": false}
INFO:tests.common.plugins.sanity_check:No post-test check is required. Done post-test sanity check
INFO:root:Can not get Allure report URL. Please check logs
pkt = <Ether dst=22:2f:c5:5b:bb:b0 src=00:06:07:08:09:0a type=IPv4 |<IP ihl=None tos=0x0 id=1 ttl=64 proto=hopopt src=192....\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&'()+,-./0123456789:;<=>?@A' |>>>
pkt_list = [<Ether dst=22:2f:c5:5b:bb:b0 src=00:06:07:08:09:0a type=IPv4 |<IP ihl=None tos=0x0 id=1 ttl=64 proto=hopopt src=192...x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&'()+,-./0123456789:;<=>?@A' |>>>]
ptfadapter = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
rx_port = 27
rx_ports = {4: 27, 6: 19}
tx_port = 12
/usr/local/lib/python3.8/dist-packages/ptf/testutils.py:3270: in verify_no_packet
test.fail(
device = 0
pkt = <ptf.mask.Mask object at 0x7fa27e4e5370>
port = 27
port_id = 27
result = PollSuccess(device=0, port=27, packet=b'&\xc34\x156\xf7"/\xc5[\xbb\xb0\x08\x00E\x00\x00V\x00\x01\x00\x00?\x00^\xfc\xc0...1\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&'()+,-./0123456789:;<=>?@A', time=1725572735.6385055)
test = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
timeout = 0.1

---

self = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
msg = 'Received packet that we expected not to receive on device 0, port 27.\n========== RECEIVED ==========\n0000 26 C3 34...3C 3D ./0123456789:;<=\n0060 3E 3F 40 41 >?@A\n==============================\n'

def fail(self, msg=None):
    """Fail immediately, with the given message."""

  raise self.failureException(msg)

E AssertionError: Received packet that we expected not to receive on device 0, port 27.
E ========== RECEIVED ==========
E 0000 26 C3 34 15 36 F7 22 2F C5 5B BB B0 08 00 45 00 &.4.6."/.[....E.
E 0010 00 56 00 01 00 00 3F 00 5E FC C0 A8 00 02 5B 00 .V....?.^.....[.
E 0020 01 01 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D ................
E 0030 0E 0F 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D ................
E 0040 1E 1F 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D .. !"#$%&'()*+,-
E 0050 2E 2F 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D ./0123456789:;<=
E 0060 3E 3F 40 41 >?@A
E ==============================

msg = 'Received packet that we expected not to receive on device 0, port 27.\n========== RECEIVED ==========\n0000 26 C3 34...3C 3D ./0123456789:;<=\n0060 3E 3F 40 41 >?@A\n==============================\n'
self = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>

/usr/lib/python3.8/unittest/case.py:753: AssertionError
=============================== warnings summary ===============================
../../../../usr/local/lib/python3.8/dist-packages/paramiko/transport.py:236
/usr/local/lib/python3.8/dist-packages/paramiko/transport.py:236: CryptographyDeprecationWarning: Blowfish has been deprecated
"class": algorithms.Blowfish,

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html

• generated xml file: /var/src/sonic-mgmt/tests/logs/bgp/test_bgp_suppress_fib|||2.xml -
  =========================== short test summary info ============================
  FAILED bgp/test_bgp_suppress_fib.py::test_bgp_route_with_suppress[default] - ...
  !!!!!!!!!!!!!!!!!!!!!!!!!! stopping after 1 failures !!!!!!!!!!!!!!!!!!!!!!!!!!!
  =================== 1 failed, 1 warning in 556.37s (0:09:16) ===================

Results you expected to see

test can pass

Is it platform specific

generic

Relevant log output

No response

Output of show version

No response

Attach files (if any)

No response