Test Automation for clusterwide encryption keyrotation

Signed-off-by: Parag Kamble <[email protected]>

ocs_ci/framework/pytest_customization/marks.py

@@ -717,3 +717,9 @@ def get_current_test_marks():
 # Mark for Multicluster upgrade scenarios
 config_index = pytest.mark.config_index
 multicluster_roles = pytest.mark.multicluster_roles
+
+# Marks to identify if Vault KMS deployment is required
+vault_kms_deployment_required = pytest.mark.skipif(
+    config.ENV_DATA.get("KMS_PROVIDER", "").upper() != "VAULT",
+    reason="This test requires Vault KMS deployment.",
+)

ocs_ci/helpers/keyrotation_helper.py

@@ -8,7 +8,7 @@
 from ocs_ci.ocs.resources.pvc import get_deviceset_pvcs
 from ocs_ci.ocs.exceptions import UnexpectedBehaviour
 from ocs_ci.utility.retry import retry
-from ocs_ci.utility.kms import get_kms_details
+from ocs_ci.utility.kms import get_kms_details, is_kms_enabled
 
 log = logging.getLogger(__name__)
 
@@ -109,11 +109,7 @@ def enable_keyrotation(self):
         Returns:
             bool: True if key rotation is enabled, False otherwise.
         """
-        if self.is_keyrotation_enable():
-            log.info("Keyrotation is Already in Enabled state.")
-            return True
-
-        param = '[{"op":"remove","path":"/spec/encryption/keyRotation/enable"}]'
+        param = '[{"op": "add", "path": "/spec/encryption/keyRotation/enable", "value": true}]'
         self.storagecluster_obj.patch(params=param, format_type="json")
         resource_status = self.storagecluster_obj.wait_for_resource(
             constants.STATUS_READY,
@@ -276,12 +272,27 @@ def __init__(self):
         super().__init__()
         self.deviceset = self._get_deviceset()
 
+        # get the kms config for the OSD keyrotation
+        if is_kms_enabled(dont_raise=True) and (
+            config.ENV_DATA.get("KMS_PROVIDER")
+            in [constants.VAULT_KMS_PROVIDER, constants.HPCS_KMS_PROVIDER]
+        ):
+            self.kms = get_kms_details()
+
     def _get_deviceset(self):
         """
         Listing deviceset for OSD.
         """
         return [pvc.name for pvc in get_deviceset_pvcs()]
 
+    def enable_osd_keyrotatio(self):
+        """Enable OSD keyrotation in storagecluster Spec.
+
+        Returns:
+            bool: True if keyrotation is Enabled otherwise False
+        """
+        return self.enable_keyrotation()
+
     def is_osd_keyrotation_enabled(self):
         """
         Checks if key rotation is enabled for OSD.
@@ -364,6 +375,55 @@ def compare_old_with_new_keys():
         log.info("Keyrotation is sucessfully done for the all OSD.")
         return True
 
+    def verify_osd_keyrotation_for_kms(self, tries=10, delay=10):
+        """_summary_
+
+        Raises:
+            UnexpectedBehaviour: _description_
+
+        Returns:
+            _type_: _description_
+        """
+
+        old_keys = {}
+
+        for dev in self.deviceset:
+            old_keys[dev] = self.kms.get_osd_secret(dev)
+
+        # Noobaa Secret
+        old_keys[constants.NOOBAA_BACKEND_SECRET] = self.kms.get_noobaa_secret()
+
+        log.info(f"OSD and NooBaa keys before Rotation : {old_keys}")
+
+        @retry(UnexpectedBehaviour, tries=tries, delay=delay)
+        def compare_keys():
+            new_keys = {}
+            for dev in self.deviceset:
+                new_keys[dev] = self.kms.get_osd_secret(dev)
+
+            new_keys[constants.NOOBAA_BACKEND_SECRET] = self.kms.get_noobaa_secret()
+
+            unmatched_keys = []
+            for key in old_keys:
+                if old_keys[key] == new_keys[key]:
+                    log.info(f"Vault key for {key} is not yet rotated ")
+                    unmatched_keys.append(key)
+
+            if unmatched_keys:
+                raise UnexpectedBehaviour(
+                    f"These component keys are not rotated in vault : {','.join(unmatched_keys)}"
+                )
+
+            log.info(f"New OSD and Noobaa keys are rotated : {new_keys}")
+
+        try:
+            compare_keys()
+        except UnexpectedBehaviour:
+            log.info("OSD and Noobaa  Keys are not rotated.")
+            return False
+
+        return True
+
 
 class PVKeyrotation(KeyRotation):
     def __init__(self, sc_obj):

ocs_ci/utility/kms.py

@@ -1298,6 +1298,85 @@ def find_passphrase(obj):
 
         return secret
 
+    def get_osd_secret(self, device_handle):
+        """Fetch the OSD encryption key for the given device handle from Vault.
+
+        Args:
+            device_handle (str): The device handle for which to retrieve the OSD secret.
+
+        Returns:
+            str: The OSD encryption secret if found, otherwise None.
+        """
+        if not self.vault_backend_path:
+            self.get_vault_backend_path()
+
+        secret_key = f"rook-ceph-osd-encryption-key-{device_handle}"
+
+        # Construct the Vault command
+        cmd = f"vault kv get -format=json {self.vault_backend_path}/{secret_key}"
+
+        try:
+            # Execute the command and capture the output
+            out = subprocess.check_output(
+                shlex.split(cmd), stderr=subprocess.STDOUT, text=True
+            )
+
+            # Parse the JSON response
+            json_out = json.loads(out)
+
+            # Retrieve the secret
+            # secret_key = f"rook-ceph-osd-encryption-key-{device_handle}"
+            secret = json_out.get("data", {}).get(secret_key)
+
+            if not secret:
+                logger.error(
+                    f"Secret for key '{secret_key}' not found in Vault response."
+                )
+                return None
+
+            return secret
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Error executing Vault command: {e.output.strip()}")
+        except json.JSONDecodeError as e:
+            logger.error(f"Invalid JSON output from Vault command. : {e}")
+
+        return None
+
+    def get_noobaa_secret(self):
+        """Fetches the NooBaa backend secret from the Vault.
+
+        Returns:
+            str: The NooBaa backend secret.
+        """
+        # Construct the Vault command
+        cmd = f"vault kv get -format=json {self.vault_backend_path}/{constants.NOOBAA_BACKEND_SECRET}"
+
+        try:
+            # Execute the command and capture the output
+            out = subprocess.check_output(
+                shlex.split(cmd), stderr=subprocess.STDOUT, text=True
+            )
+
+            # Parse the JSON response
+            json_out = json.loads(out)
+
+            # Retrieve the secret
+            secret = json_out["data"].get(json_out["data"].get("active_root_key"))
+
+            if not secret:
+                logger.error(
+                    f"Secret for key '{constants.NOOBAA_BACKEND_SECRET}' not found in Vault response."
+                )
+                return None
+
+            return secret
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Error executing Vault command: {e.output.strip()}")
+        except json.JSONDecodeError as e:
+            logger.error(f"Invalid JSON output from Vault command. {e}")
+
+        return None
+
 
 class HPCS(KMS):
     """

tests/functional/encryption/test_encryption_keyrotation.py

@@ -1,12 +1,14 @@
 import logging
 import pytest
+import json
 
 from ocs_ci.framework.pytest_customization.marks import (
     tier1,
     green_squad,
     encryption_at_rest_required,
     skipif_kms_deployment,
     skipif_external_mode,
+    vault_kms_deployment_required,
 )
 from ocs_ci.helpers.keyrotation_helper import (
     NoobaaKeyrotation,
@@ -16,6 +18,9 @@
 
 from ocs_ci.ocs.exceptions import UnexpectedBehaviour
 from ocs_ci.utility.retry import retry
+from ocs_ci.ocs import constants
+from ocs_ci.helpers.helpers import create_pods
+from concurrent.futures import ThreadPoolExecutor
 
 
 log = logging.getLogger(__name__)
@@ -230,3 +235,109 @@ def compare_old_keys_with_new_keys():
         # Change the keyrotation value to default.
         log.info("Changing the keyrotation value to default.")
         noobaa_keyrotation.set_keyrotation_schedule("@weekly")
+
+
+@green_squad
+@tier1
+@encryption_at_rest_required
+@vault_kms_deployment_required
+@skipif_external_mode
+class TestOSDKeyrotationWithKMS:
+    @pytest.fixture(autouse=True)
+    def setup(
+        self,
+    ):
+        self.keyrotation = OSDKeyrotation()
+        preserve_encryption_status = (
+            self.keyrotation.storagecluster_obj.data["spec"]
+            .get("encryption")
+            .get("keyRotation")
+        )
+        self.keyrotation.set_keyrotation_schedule("*/2 * * * *")
+        self.keyrotation.enable_keyrotation()
+        yield
+        if preserve_encryption_status:
+            param = json.dumps(
+                [
+                    {
+                        "op": "add",
+                        "path": "/spec/encryption/keyRotation",
+                        "value": preserve_encryption_status,
+                    }
+                ]
+            )
+            self.keyrotation.storagecluster_obj.patch(params=param, format_type="json")
+            self.keyrotation.storagecluster_obj.wait_for_resource(
+                constants.STATUS_READY,
+                self.keyrotation.storagecluster_obj.resource_name,
+                column="PHASE",
+                timeout=180,
+            )
+
+    def test_osd_keyrotation_with_kms(self, multi_pvc_factory, pod_factory):
+        """Test OSD KEyrotation operation for vault KMS.
+
+        Steps:
+            1. Deploy cluster with clusterwide encryption
+            2. enable keyrotation and set keyrotation schedule for every 2 minute.
+            3. create a multiple PVC and attach it to the pod.
+            4. start IO's on PVC to pods.
+            5. Verify keyrotation operation happening for every 2 minutes.
+
+        """
+        size = 5
+        access_modes = {
+            constants.CEPHBLOCKPOOL: [
+                f"{constants.ACCESS_MODE_RWO}-Block",
+                f"{constants.ACCESS_MODE_RWX}-Block",
+            ],
+            constants.CEPHFILESYSTEM: [
+                constants.ACCESS_MODE_RWO,
+                constants.ACCESS_MODE_RWX,
+            ],
+        }
+
+        # Create PVCs for CephBlockPool and CephFS
+        pvc_objects = {
+            interface: multi_pvc_factory(
+                interface=interface,
+                access_modes=modes,
+                size=size,
+                num_of_pvc=2,
+            )
+            for interface, modes in access_modes.items()
+        }
+
+        # Create pods for each interface
+        self.all_pods = []
+        for interface, pvcs in pvc_objects.items():
+            pods = create_pods(
+                pvc_objs=pvcs,
+                pod_factory=pod_factory,
+                interface=interface,
+                pods_for_rwx=2,  # Create 2 pods for each RWX PVC
+                status=constants.STATUS_RUNNING,
+            )
+            assert pods, f"Failed to create pods for {interface}."
+            log.info(f"Created {len(pods)} pods for interface: {interface}")
+            self.all_pods.extend(pods)
+
+        # Perform I/O on all pods using ThreadPoolExecutor
+        log.info("Starting I/O operations on all pods.")
+        with ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(
+                    pod_obj.run_io, storage_type="fs", size="1G", runtime=60
+                )
+                for pod_obj in self.all_pods
+            ]
+
+            log.info("Verifying OSD keyrotation for KMS.")
+            assert self.keyrotation.verify_osd_keyrotation_for_kms(
+                tries=6, delay=10
+            ), "Failed to rotate OSD and NooBaa Keys in KMS."
+            log.info("Keyrotation verification successful.")
+
+            # Wait for I/O operations to complete
+            for future in futures:
+                future.result()