From 293b837dc3003220d5ab33e83689a5fe4c61dc55 Mon Sep 17 00:00:00 2001 From: nicholasyang Date: Fri, 6 Sep 2024 11:24:49 +0800 Subject: [PATCH] Dev: ui_corosync: use corosync-cfgtool instead of corosync-cmaptool to retreive link status (jsc#PED-8083) as the data provided in `corosync-cmaptool -m stats` in not accurate in certain cases. --- crmsh/ui_corosync.py | 27 ++-- test/features/corosync_ui.feature | 3 +- test/unittests/test_ui_corosync.py | 225 ++++++++++++++++++++--------- 3 files changed, 171 insertions(+), 84 deletions(-) diff --git a/crmsh/ui_corosync.py b/crmsh/ui_corosync.py index f79ff8886..39cc9d5ee 100644 --- a/crmsh/ui_corosync.py +++ b/crmsh/ui_corosync.py @@ -1,6 +1,7 @@ # Copyright (C) 2013 Kristoffer Gronlund # See COPYING for license information. import dataclasses +import io import ipaddress import json import re @@ -234,7 +235,7 @@ def _check_link_removable(lm: corosync.LinkManager, linknumber): if not ServiceManager().service_is_active("corosync.service"): return nodes = utils.list_cluster_nodes() - for host, result in prun.prun({node: 'corosync-cmapctl -m stats' for node in nodes}).items(): + for host, result in prun.prun({node: 'corosync-cfgtool -s' for node in nodes}).items(): # for each node in the cluster match result: case prun.SSHError() as e: @@ -242,16 +243,20 @@ def _check_link_removable(lm: corosync.LinkManager, linknumber): case prun.ProcessResult() as x: if x.returncode != 0: raise ValueError(f'{host}: {x.returncode}, {sh.Utils.decode_str(x.stderr)}') - stdout = sh.Utils.decode_str(x.stdout) - connected_nodes = { - nodeid - for nodeid, ln, connected in re.findall( - '^stats\\.knet\\.node([0-9]+)\\.link([0-9]+)\\.connected\\W+.*=\\s*([0-9]+)$', - stdout, re.ASCII | re.MULTILINE, - ) if connected == '1' # only connected node - and ln != str(linknumber) # filter out the link to be removed - } - # the number of nodes connected to should be equal to the total number of nodes in the cluster + connected_nodes = set() + ln = -1 + for line in io.StringIO(sh.Utils.decode_str(x.stdout)): + mo = re.match('^(?:LINK ID (\\d+)|\\s*nodeid:\\s*(\\d+):\\s*(?:localhost|connected)$)', line) + if mo is None: + continue + if mo.group(1): + ln = int(mo.group(1)) + continue + if ln == linknumber: + # filter out the link to be removed + continue + if mo.group(2): + connected_nodes.add(mo.group(2)) if len(connected_nodes) < len(next(link for link in lm.links() if link is not None).nodes): # or will lose quorum raise ValueError(f'Cannot remove link {linknumber}. Removing this link makes the cluster to lose quorum.') diff --git a/test/features/corosync_ui.feature b/test/features/corosync_ui.feature index 3ab5c9c9c..2449e1926 100644 --- a/test/features/corosync_ui.feature +++ b/test/features/corosync_ui.feature @@ -37,7 +37,8 @@ Feature: crm corosync ui test cases When Run "crm corosync link update 0 hanode1=@hanode1.ip.1 hanode2=@hanode2.ip.1 options knet_link_priority=10" on "hanode1" Then Expected "Restarting corosync.service is needed to apply the changes, ie. crm cluster restart --all" in stderr Given Run "systemctl restart corosync.service" OK on "hanode1,hanode2" - When Try "crm corosync link add hanode1=@hanode1.ip.0 hanode2=@hanode2.ip.0 options knet_link_priority=" on "hanode1" + When Wait "5" seconds + And Try "crm corosync link add hanode1=@hanode1.ip.0 hanode2=@hanode2.ip.0 options knet_link_priority=" on "hanode1" Then Expected "invalid option" in stderr Given Run "crm corosync link add hanode1=@hanode1.ip.0 hanode2=@hanode2.ip.0 options knet_link_priority=11" OK on "hanode1" When Try "crm corosync link update 1 hanode1=@hanode1.ip.1" on "hanode1" diff --git a/test/unittests/test_ui_corosync.py b/test/unittests/test_ui_corosync.py index e5ffc8747..5e9d5ccb4 100644 --- a/test/unittests/test_ui_corosync.py +++ b/test/unittests/test_ui_corosync.py @@ -109,30 +109,51 @@ def test_all_links_connected(self, mock_sia, mock_lcn, mock_prun): mock_prun.return_value = { 'node1': prun.ProcessResult( returncode=0, stderr=b'', - stdout=b'stats.knet.node1.link0.connected (u8) = 1\n' - b'stats.knet.node2.link0.connected (u8) = 1\n' - b'stats.knet.node3.link0.connected (u8) = 1\n' - b'stats.knet.node1.link1.connected (u8) = 1\n' - b'stats.knet.node2.link1.connected (u8) = 1\n' - b'stats.knet.node3.link1.connected (u8) = 1\n' + stdout=b'Local node ID 1, transport knet\n' + b'LINK ID 0 udp\n' + b' addr = 192.0.2.100\n' + b' status:\n' + b' nodeid: 1: localhost\n' + b' nodeid: 2: connected\n' + b' nodeid: 3: connected\n' + b'LINK ID 1 udp\n' + b' addr = 192.0.2.200\n' + b' status:\n' + b' nodeid: 1: localhost\n' + b' nodeid: 2: connected\n' + b' nodeid: 3: connected\n', ), 'node2': prun.ProcessResult( returncode=0, stderr=b'', - stdout=b'stats.knet.node1.link0.connected (u8) = 1\n' - b'stats.knet.node2.link0.connected (u8) = 1\n' - b'stats.knet.node3.link0.connected (u8) = 1\n' - b'stats.knet.node1.link1.connected (u8) = 1\n' - b'stats.knet.node2.link1.connected (u8) = 1\n' - b'stats.knet.node3.link1.connected (u8) = 1\n' + stdout=b'Local node ID 1, transport knet\n' + b'LINK ID 0 udp\n' + b' addr = 192.0.2.101\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: localhost\n' + b' nodeid: 3: connected\n' + b'LINK ID 1 udp\n' + b' addr = 192.0.2.201\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: localhost\n' + b' nodeid: 3: connected\n', ), 'node3': prun.ProcessResult( returncode=0, stderr=b'', - stdout=b'stats.knet.node1.link0.connected (u8) = 1\n' - b'stats.knet.node2.link0.connected (u8) = 1\n' - b'stats.knet.node3.link0.connected (u8) = 1\n' - b'stats.knet.node1.link1.connected (u8) = 1\n' - b'stats.knet.node2.link1.connected (u8) = 1\n' - b'stats.knet.node3.link1.connected (u8) = 1\n' + stdout=b'Local node ID 1, transport knet\n' + b'LINK ID 0 udp\n' + b' addr = 192.0.2.102\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: connected\n' + b' nodeid: 3: localhost\n' + b'LINK ID 1 udp\n' + b' addr = 192.0.2.202\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: connected\n' + b' nodeid: 3: localhost\n', ), } self.mock_lm.links.return_value = [ @@ -156,30 +177,51 @@ def test_one_node_pair_disconnected(self, mock_sia, mock_lcn, mock_prun): mock_prun.return_value = { 'node1': prun.ProcessResult( returncode=0, stderr=b'', - stdout=b'stats.knet.node1.link0.connected (u8) = 1\n' - b'stats.knet.node2.link0.connected (u8) = 1\n' - b'stats.knet.node3.link0.connected (u8) = 1\n' - b'stats.knet.node1.link1.connected (u8) = 1\n' - b'stats.knet.node2.link1.connected (u8) = 1\n' - b'stats.knet.node3.link1.connected (u8) = 1\n' + stdout=b'Local node ID 1, transport knet\n' + b'LINK ID 0 udp\n' + b' addr = 192.0.2.100\n' + b' status:\n' + b' nodeid: 1: localhost\n' + b' nodeid: 2: connected\n' + b' nodeid: 3: connected\n' + b'LINK ID 1 udp\n' + b' addr = 192.0.2.200\n' + b' status:\n' + b' nodeid: 1: localhost\n' + b' nodeid: 2: connected\n' + b' nodeid: 3: connected\n', ), 'node2': prun.ProcessResult( returncode=0, stderr=b'', - stdout=b'stats.knet.node1.link0.connected (u8) = 1\n' - b'stats.knet.node2.link0.connected (u8) = 1\n' - b'stats.knet.node3.link0.connected (u8) = 1\n' - b'stats.knet.node1.link1.connected (u8) = 1\n' - b'stats.knet.node2.link1.connected (u8) = 1\n' - b'stats.knet.node3.link1.connected (u8) = 0\n' + stdout=b'Local node ID 1, transport knet\n' + b'LINK ID 0 udp\n' + b' addr = 192.0.2.101\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: localhost\n' + b' nodeid: 3: connected\n' + b'LINK ID 1 udp\n' + b' addr = 192.0.2.201\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: localhost\n' + b' nodeid: 3: disconnected\n', ), 'node3': prun.ProcessResult( returncode=0, stderr=b'', - stdout=b'stats.knet.node1.link0.connected (u8) = 1\n' - b'stats.knet.node2.link0.connected (u8) = 1\n' - b'stats.knet.node3.link0.connected (u8) = 1\n' - b'stats.knet.node1.link1.connected (u8) = 1\n' - b'stats.knet.node2.link1.connected (u8) = 0\n' - b'stats.knet.node3.link1.connected (u8) = 1\n' + stdout=b'Local node ID 1, transport knet\n' + b'LINK ID 0 udp\n' + b' addr = 192.0.2.102\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: connected\n' + b' nodeid: 3: localhost\n' + b'LINK ID 1 udp\n' + b' addr = 192.0.2.202\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: disconnected\n' + b' nodeid: 3: localhost\n', ), } self.mock_lm.links.return_value = [ @@ -204,48 +246,87 @@ def test_pair_connected(self, mock_sia, mock_lcn, mock_prun): mock_prun.return_value = { 'node1': prun.ProcessResult( returncode=0, stderr=b'', - stdout=b'stats.knet.node1.link0.connected (u8) = 1\n' - b'stats.knet.node2.link0.connected (u8) = 1\n' - b'stats.knet.node3.link0.connected (u8) = 1\n' - b'stats.knet.node1.link1.connected (u8) = 1\n' - b'stats.knet.node2.link1.connected (u8) = 1\n' - b'stats.knet.node3.link1.connected (u8) = 0\n' - b'stats.knet.node1.link2.connected (u8) = 1\n' - b'stats.knet.node2.link2.connected (u8) = 0\n' - b'stats.knet.node3.link2.connected (u8) = 0\n' - b'stats.knet.node1.link3.connected (u8) = 1\n' - b'stats.knet.node2.link3.connected (u8) = 0\n' - b'stats.knet.node3.link3.connected (u8) = 1\n' + stdout=b'Local node ID 1, transport knet\n' + b'LINK ID 0 udp\n' + b' addr = 192.0.2.100\n' + b' status:\n' + b' nodeid: 1: localhost\n' + b' nodeid: 2: connected\n' + b' nodeid: 3: connected\n' + b'LINK ID 1 udp\n' + b' addr = 192.0.2.100\n' + b' status:\n' + b' nodeid: 1: localhost\n' + b' nodeid: 2: connected\n' + b' nodeid: 3: disconnected\n' + b'LINK ID 2 udp\n' + b' addr = 192.0.2.100\n' + b' status:\n' + b' nodeid: 1: localhost\n' + b' nodeid: 2: disconnected\n' + b' nodeid: 3: disconnected\n' + b'LINK ID 3 udp\n' + b' addr = 192.0.2.100\n' + b' status:\n' + b' nodeid: 1: localhost\n' + b' nodeid: 2: disconnected\n' + b' nodeid: 3: connected\n' ), 'node2': prun.ProcessResult( returncode=0, stderr=b'', - stdout=b'stats.knet.node1.link0.connected (u8) = 1\n' - b'stats.knet.node2.link0.connected (u8) = 1\n' - b'stats.knet.node3.link0.connected (u8) = 1\n' - b'stats.knet.node1.link1.connected (u8) = 1\n' - b'stats.knet.node2.link1.connected (u8) = 1\n' - b'stats.knet.node3.link1.connected (u8) = 0\n' - b'stats.knet.node1.link2.connected (u8) = 0\n' - b'stats.knet.node2.link2.connected (u8) = 1\n' - b'stats.knet.node3.link2.connected (u8) = 1\n' - b'stats.knet.node1.link3.connected (u8) = 0\n' - b'stats.knet.node2.link3.connected (u8) = 1\n' - b'stats.knet.node3.link3.connected (u8) = 0\n' + stdout=b'Local node ID 1, transport knet\n' + b'LINK ID 0 udp\n' + b' addr = 192.0.2.100\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: localhost\n' + b' nodeid: 3: connected\n' + b'LINK ID 1 udp\n' + b' addr = 192.0.2.101\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: localhost\n' + b' nodeid: 3: disconnected\n' + b'LINK ID 2 udp\n' + b' addr = 192.0.2.101\n' + b' status:\n' + b' nodeid: 1: disconnected\n' + b' nodeid: 2: localhost\n' + b' nodeid: 3: connected\n' + b'LINK ID 3 udp\n' + b' addr = 192.0.2.101\n' + b' status:\n' + b' nodeid: 1: disconnected\n' + b' nodeid: 2: localhost\n' + b' nodeid: 3: disconnected\n' ), 'node3': prun.ProcessResult( returncode=0, stderr=b'', - stdout=b'stats.knet.node1.link0.connected (u8) = 1\n' - b'stats.knet.node2.link0.connected (u8) = 1\n' - b'stats.knet.node3.link0.connected (u8) = 1\n' - b'stats.knet.node1.link1.connected (u8) = 0\n' - b'stats.knet.node2.link1.connected (u8) = 0\n' - b'stats.knet.node3.link1.connected (u8) = 1\n' - b'stats.knet.node1.link2.connected (u8) = 0\n' - b'stats.knet.node2.link2.connected (u8) = 1\n' - b'stats.knet.node3.link2.connected (u8) = 1\n' - b'stats.knet.node1.link3.connected (u8) = 1\n' - b'stats.knet.node2.link3.connected (u8) = 0\n' - b'stats.knet.node3.link3.connected (u8) = 1\n' + stdout=b'Local node ID 1, transport knet\n' + b'LINK ID 0 udp\n' + b' addr = 192.0.2.100\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: connected\n' + b' nodeid: 3: localhost\n' + b'LINK ID 1 udp\n' + b' addr = 192.0.2.102\n' + b' status:\n' + b' nodeid: 1: disconnected\n' + b' nodeid: 2: disconnected\n' + b' nodeid: 3: localhost\n' + b'LINK ID 2 udp\n' + b' addr = 192.0.2.102\n' + b' status:\n' + b' nodeid: 1: disconnected\n' + b' nodeid: 2: connected\n' + b' nodeid: 3: localhost\n' + b'LINK ID 3 udp\n' + b' addr = 192.0.2.102\n' + b' status:\n' + b' nodeid: 1: connected\n' + b' nodeid: 2: disconnected\n' + b' nodeid: 3: localhost\n' ), } self.mock_lm.links.return_value = [