Skip to content

Commit

Permalink
Dev: ui_corosync: use corosync-cfgtool instead of corosync-cmaptool t…
Browse files Browse the repository at this point in the history 
…o retreive link status (jsc#PED-8083)

as the data provided in `corosync-cmaptool -m stats` in not accurate in
certain cases.
  • Loading branch information
@nicholasyang2022
nicholasyang2022 committed Sep 6, 2024
1 parent 12bb26e commit 293b837
Show file tree
Hide file tree
Showing 3 changed files with 171 additions and 84 deletions.
27 changes: 16 additions & 11 deletions crmsh/ui_corosync.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Copyright (C) 2013 Kristoffer Gronlund <[email protected]>
# See COPYING for license information.
import dataclasses
import io
import ipaddress
import json
import re
Expand Down Expand Up @@ -234,24 +235,28 @@ def _check_link_removable(lm: corosync.LinkManager, linknumber):
if not ServiceManager().service_is_active("corosync.service"):
return
nodes = utils.list_cluster_nodes()
for host, result in prun.prun({node: 'corosync-cmapctl -m stats' for node in nodes}).items():
for host, result in prun.prun({node: 'corosync-cfgtool -s' for node in nodes}).items():
# for each node <host> in the cluster
match result:
case prun.SSHError() as e:
raise ValueError(str(e))
case prun.ProcessResult() as x:
if x.returncode != 0:
raise ValueError(f'{host}: {x.returncode}, {sh.Utils.decode_str(x.stderr)}')
stdout = sh.Utils.decode_str(x.stdout)
connected_nodes = {
nodeid
for nodeid, ln, connected in re.findall(
'^stats\\.knet\\.node([0-9]+)\\.link([0-9]+)\\.connected\\W+.*=\\s*([0-9]+)$',
stdout, re.ASCII | re.MULTILINE,
) if connected == '1' # only connected node
and ln != str(linknumber) # filter out the link to be removed
}
# the number of nodes connected to <host> should be equal to the total number of nodes in the cluster
connected_nodes = set()
ln = -1
for line in io.StringIO(sh.Utils.decode_str(x.stdout)):
mo = re.match('^(?:LINK ID (\\d+)|\\s*nodeid:\\s*(\\d+):\\s*(?:localhost|connected)$)', line)
if mo is None:
continue
if mo.group(1):
ln = int(mo.group(1))
continue
if ln == linknumber:
# filter out the link to be removed
continue
if mo.group(2):
connected_nodes.add(mo.group(2))
if len(connected_nodes) < len(next(link for link in lm.links() if link is not None).nodes):
# or <host> will lose quorum
raise ValueError(f'Cannot remove link {linknumber}. Removing this link makes the cluster to lose quorum.')
Expand Down
3 changes: 2 additions & 1 deletion test/features/corosync_ui.feature
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ Feature: crm corosync ui test cases
When Run "crm corosync link update 0 [email protected] [email protected] options knet_link_priority=10" on "hanode1"
Then Expected "Restarting corosync.service is needed to apply the changes, ie. crm cluster restart --all" in stderr
Given Run "systemctl restart corosync.service" OK on "hanode1,hanode2"
When Try "crm corosync link add [email protected] [email protected] options knet_link_priority=" on "hanode1"
When Wait "5" seconds
And Try "crm corosync link add [email protected] [email protected] options knet_link_priority=" on "hanode1"
Then Expected "invalid option" in stderr
Given Run "crm corosync link add [email protected] [email protected] options knet_link_priority=11" OK on "hanode1"
When Try "crm corosync link update 1 [email protected]" on "hanode1"
Expand Down
225 changes: 153 additions & 72 deletions test/unittests/test_ui_corosync.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,30 +109,51 @@ def test_all_links_connected(self, mock_sia, mock_lcn, mock_prun):
mock_prun.return_value = {
'node1': prun.ProcessResult(
returncode=0, stderr=b'',
stdout=b'stats.knet.node1.link0.connected (u8) = 1\n'
b'stats.knet.node2.link0.connected (u8) = 1\n'
b'stats.knet.node3.link0.connected (u8) = 1\n'
b'stats.knet.node1.link1.connected (u8) = 1\n'
b'stats.knet.node2.link1.connected (u8) = 1\n'
b'stats.knet.node3.link1.connected (u8) = 1\n'
stdout=b'Local node ID 1, transport knet\n'
b'LINK ID 0 udp\n'
b' addr = 192.0.2.100\n'
b' status:\n'
b' nodeid: 1: localhost\n'
b' nodeid: 2: connected\n'
b' nodeid: 3: connected\n'
b'LINK ID 1 udp\n'
b' addr = 192.0.2.200\n'
b' status:\n'
b' nodeid: 1: localhost\n'
b' nodeid: 2: connected\n'
b' nodeid: 3: connected\n',
),
'node2': prun.ProcessResult(
returncode=0, stderr=b'',
stdout=b'stats.knet.node1.link0.connected (u8) = 1\n'
b'stats.knet.node2.link0.connected (u8) = 1\n'
b'stats.knet.node3.link0.connected (u8) = 1\n'
b'stats.knet.node1.link1.connected (u8) = 1\n'
b'stats.knet.node2.link1.connected (u8) = 1\n'
b'stats.knet.node3.link1.connected (u8) = 1\n'
stdout=b'Local node ID 1, transport knet\n'
b'LINK ID 0 udp\n'
b' addr = 192.0.2.101\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: localhost\n'
b' nodeid: 3: connected\n'
b'LINK ID 1 udp\n'
b' addr = 192.0.2.201\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: localhost\n'
b' nodeid: 3: connected\n',
),
'node3': prun.ProcessResult(
returncode=0, stderr=b'',
stdout=b'stats.knet.node1.link0.connected (u8) = 1\n'
b'stats.knet.node2.link0.connected (u8) = 1\n'
b'stats.knet.node3.link0.connected (u8) = 1\n'
b'stats.knet.node1.link1.connected (u8) = 1\n'
b'stats.knet.node2.link1.connected (u8) = 1\n'
b'stats.knet.node3.link1.connected (u8) = 1\n'
stdout=b'Local node ID 1, transport knet\n'
b'LINK ID 0 udp\n'
b' addr = 192.0.2.102\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: connected\n'
b' nodeid: 3: localhost\n'
b'LINK ID 1 udp\n'
b' addr = 192.0.2.202\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: connected\n'
b' nodeid: 3: localhost\n',
),
}
self.mock_lm.links.return_value = [
Expand All @@ -156,30 +177,51 @@ def test_one_node_pair_disconnected(self, mock_sia, mock_lcn, mock_prun):
mock_prun.return_value = {
'node1': prun.ProcessResult(
returncode=0, stderr=b'',
stdout=b'stats.knet.node1.link0.connected (u8) = 1\n'
b'stats.knet.node2.link0.connected (u8) = 1\n'
b'stats.knet.node3.link0.connected (u8) = 1\n'
b'stats.knet.node1.link1.connected (u8) = 1\n'
b'stats.knet.node2.link1.connected (u8) = 1\n'
b'stats.knet.node3.link1.connected (u8) = 1\n'
stdout=b'Local node ID 1, transport knet\n'
b'LINK ID 0 udp\n'
b' addr = 192.0.2.100\n'
b' status:\n'
b' nodeid: 1: localhost\n'
b' nodeid: 2: connected\n'
b' nodeid: 3: connected\n'
b'LINK ID 1 udp\n'
b' addr = 192.0.2.200\n'
b' status:\n'
b' nodeid: 1: localhost\n'
b' nodeid: 2: connected\n'
b' nodeid: 3: connected\n',
),
'node2': prun.ProcessResult(
returncode=0, stderr=b'',
stdout=b'stats.knet.node1.link0.connected (u8) = 1\n'
b'stats.knet.node2.link0.connected (u8) = 1\n'
b'stats.knet.node3.link0.connected (u8) = 1\n'
b'stats.knet.node1.link1.connected (u8) = 1\n'
b'stats.knet.node2.link1.connected (u8) = 1\n'
b'stats.knet.node3.link1.connected (u8) = 0\n'
stdout=b'Local node ID 1, transport knet\n'
b'LINK ID 0 udp\n'
b' addr = 192.0.2.101\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: localhost\n'
b' nodeid: 3: connected\n'
b'LINK ID 1 udp\n'
b' addr = 192.0.2.201\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: localhost\n'
b' nodeid: 3: disconnected\n',
),
'node3': prun.ProcessResult(
returncode=0, stderr=b'',
stdout=b'stats.knet.node1.link0.connected (u8) = 1\n'
b'stats.knet.node2.link0.connected (u8) = 1\n'
b'stats.knet.node3.link0.connected (u8) = 1\n'
b'stats.knet.node1.link1.connected (u8) = 1\n'
b'stats.knet.node2.link1.connected (u8) = 0\n'
b'stats.knet.node3.link1.connected (u8) = 1\n'
stdout=b'Local node ID 1, transport knet\n'
b'LINK ID 0 udp\n'
b' addr = 192.0.2.102\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: connected\n'
b' nodeid: 3: localhost\n'
b'LINK ID 1 udp\n'
b' addr = 192.0.2.202\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: disconnected\n'
b' nodeid: 3: localhost\n',
),
}
self.mock_lm.links.return_value = [
Expand All @@ -204,48 +246,87 @@ def test_pair_connected(self, mock_sia, mock_lcn, mock_prun):
mock_prun.return_value = {
'node1': prun.ProcessResult(
returncode=0, stderr=b'',
stdout=b'stats.knet.node1.link0.connected (u8) = 1\n'
b'stats.knet.node2.link0.connected (u8) = 1\n'
b'stats.knet.node3.link0.connected (u8) = 1\n'
b'stats.knet.node1.link1.connected (u8) = 1\n'
b'stats.knet.node2.link1.connected (u8) = 1\n'
b'stats.knet.node3.link1.connected (u8) = 0\n'
b'stats.knet.node1.link2.connected (u8) = 1\n'
b'stats.knet.node2.link2.connected (u8) = 0\n'
b'stats.knet.node3.link2.connected (u8) = 0\n'
b'stats.knet.node1.link3.connected (u8) = 1\n'
b'stats.knet.node2.link3.connected (u8) = 0\n'
b'stats.knet.node3.link3.connected (u8) = 1\n'
stdout=b'Local node ID 1, transport knet\n'
b'LINK ID 0 udp\n'
b' addr = 192.0.2.100\n'
b' status:\n'
b' nodeid: 1: localhost\n'
b' nodeid: 2: connected\n'
b' nodeid: 3: connected\n'
b'LINK ID 1 udp\n'
b' addr = 192.0.2.100\n'
b' status:\n'
b' nodeid: 1: localhost\n'
b' nodeid: 2: connected\n'
b' nodeid: 3: disconnected\n'
b'LINK ID 2 udp\n'
b' addr = 192.0.2.100\n'
b' status:\n'
b' nodeid: 1: localhost\n'
b' nodeid: 2: disconnected\n'
b' nodeid: 3: disconnected\n'
b'LINK ID 3 udp\n'
b' addr = 192.0.2.100\n'
b' status:\n'
b' nodeid: 1: localhost\n'
b' nodeid: 2: disconnected\n'
b' nodeid: 3: connected\n'
),
'node2': prun.ProcessResult(
returncode=0, stderr=b'',
stdout=b'stats.knet.node1.link0.connected (u8) = 1\n'
b'stats.knet.node2.link0.connected (u8) = 1\n'
b'stats.knet.node3.link0.connected (u8) = 1\n'
b'stats.knet.node1.link1.connected (u8) = 1\n'
b'stats.knet.node2.link1.connected (u8) = 1\n'
b'stats.knet.node3.link1.connected (u8) = 0\n'
b'stats.knet.node1.link2.connected (u8) = 0\n'
b'stats.knet.node2.link2.connected (u8) = 1\n'
b'stats.knet.node3.link2.connected (u8) = 1\n'
b'stats.knet.node1.link3.connected (u8) = 0\n'
b'stats.knet.node2.link3.connected (u8) = 1\n'
b'stats.knet.node3.link3.connected (u8) = 0\n'
stdout=b'Local node ID 1, transport knet\n'
b'LINK ID 0 udp\n'
b' addr = 192.0.2.100\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: localhost\n'
b' nodeid: 3: connected\n'
b'LINK ID 1 udp\n'
b' addr = 192.0.2.101\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: localhost\n'
b' nodeid: 3: disconnected\n'
b'LINK ID 2 udp\n'
b' addr = 192.0.2.101\n'
b' status:\n'
b' nodeid: 1: disconnected\n'
b' nodeid: 2: localhost\n'
b' nodeid: 3: connected\n'
b'LINK ID 3 udp\n'
b' addr = 192.0.2.101\n'
b' status:\n'
b' nodeid: 1: disconnected\n'
b' nodeid: 2: localhost\n'
b' nodeid: 3: disconnected\n'
),
'node3': prun.ProcessResult(
returncode=0, stderr=b'',
stdout=b'stats.knet.node1.link0.connected (u8) = 1\n'
b'stats.knet.node2.link0.connected (u8) = 1\n'
b'stats.knet.node3.link0.connected (u8) = 1\n'
b'stats.knet.node1.link1.connected (u8) = 0\n'
b'stats.knet.node2.link1.connected (u8) = 0\n'
b'stats.knet.node3.link1.connected (u8) = 1\n'
b'stats.knet.node1.link2.connected (u8) = 0\n'
b'stats.knet.node2.link2.connected (u8) = 1\n'
b'stats.knet.node3.link2.connected (u8) = 1\n'
b'stats.knet.node1.link3.connected (u8) = 1\n'
b'stats.knet.node2.link3.connected (u8) = 0\n'
b'stats.knet.node3.link3.connected (u8) = 1\n'
stdout=b'Local node ID 1, transport knet\n'
b'LINK ID 0 udp\n'
b' addr = 192.0.2.100\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: connected\n'
b' nodeid: 3: localhost\n'
b'LINK ID 1 udp\n'
b' addr = 192.0.2.102\n'
b' status:\n'
b' nodeid: 1: disconnected\n'
b' nodeid: 2: disconnected\n'
b' nodeid: 3: localhost\n'
b'LINK ID 2 udp\n'
b' addr = 192.0.2.102\n'
b' status:\n'
b' nodeid: 1: disconnected\n'
b' nodeid: 2: connected\n'
b' nodeid: 3: localhost\n'
b'LINK ID 3 udp\n'
b' addr = 192.0.2.102\n'
b' status:\n'
b' nodeid: 1: connected\n'
b' nodeid: 2: disconnected\n'
b' nodeid: 3: localhost\n'
),
}
self.mock_lm.links.return_value = [
Expand Down

0 comments on commit 293b837

Please sign in to comment.