diff --git a/ocs_ci/framework/pytest_customization/marks.py b/ocs_ci/framework/pytest_customization/marks.py index 7ca6a065332..70ce7dab83a 100644 --- a/ocs_ci/framework/pytest_customization/marks.py +++ b/ocs_ci/framework/pytest_customization/marks.py @@ -717,3 +717,9 @@ def get_current_test_marks(): # Mark for Multicluster upgrade scenarios config_index = pytest.mark.config_index multicluster_roles = pytest.mark.multicluster_roles + +# Marks to identify if Vault KMS deployment is required +vault_kms_deployment_required = pytest.mark.skipif( + config.ENV_DATA.get("KMS_PROVIDER", "").upper() != "VAULT", + reason="This test requires Vault KMS deployment.", +) diff --git a/ocs_ci/helpers/keyrotation_helper.py b/ocs_ci/helpers/keyrotation_helper.py index ebea7388c81..d89c386b226 100644 --- a/ocs_ci/helpers/keyrotation_helper.py +++ b/ocs_ci/helpers/keyrotation_helper.py @@ -8,7 +8,7 @@ from ocs_ci.ocs.resources.pvc import get_deviceset_pvcs from ocs_ci.ocs.exceptions import UnexpectedBehaviour from ocs_ci.utility.retry import retry -from ocs_ci.utility.kms import get_kms_details +from ocs_ci.utility.kms import get_kms_details, is_kms_enabled log = logging.getLogger(__name__) @@ -109,11 +109,7 @@ def enable_keyrotation(self): Returns: bool: True if key rotation is enabled, False otherwise. """ - if self.is_keyrotation_enable(): - log.info("Keyrotation is Already in Enabled state.") - return True - - param = '[{"op":"remove","path":"/spec/encryption/keyRotation/enable"}]' + param = '[{"op": "add", "path": "/spec/encryption/keyRotation/enable", "value": true}]' self.storagecluster_obj.patch(params=param, format_type="json") resource_status = self.storagecluster_obj.wait_for_resource( constants.STATUS_READY, @@ -276,12 +272,27 @@ def __init__(self): super().__init__() self.deviceset = self._get_deviceset() + # get the kms config for the OSD keyrotation + if is_kms_enabled(dont_raise=True) and ( + config.ENV_DATA.get("KMS_PROVIDER") + in [constants.VAULT_KMS_PROVIDER, constants.HPCS_KMS_PROVIDER] + ): + self.kms = get_kms_details() + def _get_deviceset(self): """ Listing deviceset for OSD. """ return [pvc.name for pvc in get_deviceset_pvcs()] + def enable_osd_keyrotatio(self): + """Enable OSD keyrotation in storagecluster Spec. + + Returns: + bool: True if keyrotation is Enabled otherwise False + """ + return self.enable_keyrotation() + def is_osd_keyrotation_enabled(self): """ Checks if key rotation is enabled for OSD. @@ -364,6 +375,55 @@ def compare_old_with_new_keys(): log.info("Keyrotation is sucessfully done for the all OSD.") return True + def verify_osd_keyrotation_for_kms(self, tries=10, delay=10): + """_summary_ + + Raises: + UnexpectedBehaviour: _description_ + + Returns: + _type_: _description_ + """ + + old_keys = {} + + for dev in self.deviceset: + old_keys[dev] = self.kms.get_osd_secret(dev) + + # Noobaa Secret + old_keys[constants.NOOBAA_BACKEND_SECRET] = self.kms.get_noobaa_secret() + + log.info(f"OSD and NooBaa keys before Rotation : {old_keys}") + + @retry(UnexpectedBehaviour, tries=tries, delay=delay) + def compare_keys(): + new_keys = {} + for dev in self.deviceset: + new_keys[dev] = self.kms.get_osd_secret(dev) + + new_keys[constants.NOOBAA_BACKEND_SECRET] = self.kms.get_noobaa_secret() + + unmatched_keys = [] + for key in old_keys: + if old_keys[key] == new_keys[key]: + log.info(f"Vault key for {key} is not yet rotated ") + unmatched_keys.append(key) + + if unmatched_keys: + raise UnexpectedBehaviour( + f"These component keys are not rotated in vault : {','.join(unmatched_keys)}" + ) + + log.info(f"New OSD and Noobaa keys are rotated : {new_keys}") + + try: + compare_keys() + except UnexpectedBehaviour: + log.info("OSD and Noobaa Keys are not rotated.") + return False + + return True + class PVKeyrotation(KeyRotation): def __init__(self, sc_obj): diff --git a/ocs_ci/utility/kms.py b/ocs_ci/utility/kms.py index 10bc645efbc..1b1029ee8e1 100644 --- a/ocs_ci/utility/kms.py +++ b/ocs_ci/utility/kms.py @@ -1298,6 +1298,85 @@ def find_passphrase(obj): return secret + def get_osd_secret(self, device_handle): + """Fetch the OSD encryption key for the given device handle from Vault. + + Args: + device_handle (str): The device handle for which to retrieve the OSD secret. + + Returns: + str: The OSD encryption secret if found, otherwise None. + """ + if not self.vault_backend_path: + self.get_vault_backend_path() + + secret_key = f"rook-ceph-osd-encryption-key-{device_handle}" + + # Construct the Vault command + cmd = f"vault kv get -format=json {self.vault_backend_path}/{secret_key}" + + try: + # Execute the command and capture the output + out = subprocess.check_output( + shlex.split(cmd), stderr=subprocess.STDOUT, text=True + ) + + # Parse the JSON response + json_out = json.loads(out) + + # Retrieve the secret + # secret_key = f"rook-ceph-osd-encryption-key-{device_handle}" + secret = json_out.get("data", {}).get(secret_key) + + if not secret: + logger.error( + f"Secret for key '{secret_key}' not found in Vault response." + ) + return None + + return secret + except subprocess.CalledProcessError as e: + logger.error(f"Error executing Vault command: {e.output.strip()}") + except json.JSONDecodeError as e: + logger.error(f"Invalid JSON output from Vault command. : {e}") + + return None + + def get_noobaa_secret(self): + """Fetches the NooBaa backend secret from the Vault. + + Returns: + str: The NooBaa backend secret. + """ + # Construct the Vault command + cmd = f"vault kv get -format=json {self.vault_backend_path}/{constants.NOOBAA_BACKEND_SECRET}" + + try: + # Execute the command and capture the output + out = subprocess.check_output( + shlex.split(cmd), stderr=subprocess.STDOUT, text=True + ) + + # Parse the JSON response + json_out = json.loads(out) + + # Retrieve the secret + secret = json_out["data"].get(json_out["data"].get("active_root_key")) + + if not secret: + logger.error( + f"Secret for key '{constants.NOOBAA_BACKEND_SECRET}' not found in Vault response." + ) + return None + + return secret + except subprocess.CalledProcessError as e: + logger.error(f"Error executing Vault command: {e.output.strip()}") + except json.JSONDecodeError as e: + logger.error(f"Invalid JSON output from Vault command. {e}") + + return None + class HPCS(KMS): """ diff --git a/tests/functional/encryption/test_encryption_keyrotation.py b/tests/functional/encryption/test_encryption_keyrotation.py index 2740243ce53..44c83f6d9c4 100644 --- a/tests/functional/encryption/test_encryption_keyrotation.py +++ b/tests/functional/encryption/test_encryption_keyrotation.py @@ -1,5 +1,6 @@ import logging import pytest +import json from ocs_ci.framework.pytest_customization.marks import ( tier1, @@ -7,6 +8,7 @@ encryption_at_rest_required, skipif_kms_deployment, skipif_external_mode, + vault_kms_deployment_required, ) from ocs_ci.helpers.keyrotation_helper import ( NoobaaKeyrotation, @@ -16,6 +18,9 @@ from ocs_ci.ocs.exceptions import UnexpectedBehaviour from ocs_ci.utility.retry import retry +from ocs_ci.ocs import constants +from ocs_ci.helpers.helpers import create_pods +from concurrent.futures import ThreadPoolExecutor log = logging.getLogger(__name__) @@ -230,3 +235,109 @@ def compare_old_keys_with_new_keys(): # Change the keyrotation value to default. log.info("Changing the keyrotation value to default.") noobaa_keyrotation.set_keyrotation_schedule("@weekly") + + +@green_squad +@tier1 +@encryption_at_rest_required +@vault_kms_deployment_required +@skipif_external_mode +class TestOSDKeyrotationWithKMS: + @pytest.fixture(autouse=True) + def setup( + self, + ): + self.keyrotation = OSDKeyrotation() + preserve_encryption_status = ( + self.keyrotation.storagecluster_obj.data["spec"] + .get("encryption") + .get("keyRotation") + ) + self.keyrotation.set_keyrotation_schedule("*/2 * * * *") + self.keyrotation.enable_keyrotation() + yield + if preserve_encryption_status: + param = json.dumps( + [ + { + "op": "add", + "path": "/spec/encryption/keyRotation", + "value": preserve_encryption_status, + } + ] + ) + self.keyrotation.storagecluster_obj.patch(params=param, format_type="json") + self.keyrotation.storagecluster_obj.wait_for_resource( + constants.STATUS_READY, + self.keyrotation.storagecluster_obj.resource_name, + column="PHASE", + timeout=180, + ) + + def test_osd_keyrotation_with_kms(self, multi_pvc_factory, pod_factory): + """Test OSD KEyrotation operation for vault KMS. + + Steps: + 1. Deploy cluster with clusterwide encryption + 2. enable keyrotation and set keyrotation schedule for every 2 minute. + 3. create a multiple PVC and attach it to the pod. + 4. start IO's on PVC to pods. + 5. Verify keyrotation operation happening for every 2 minutes. + + """ + size = 5 + access_modes = { + constants.CEPHBLOCKPOOL: [ + f"{constants.ACCESS_MODE_RWO}-Block", + f"{constants.ACCESS_MODE_RWX}-Block", + ], + constants.CEPHFILESYSTEM: [ + constants.ACCESS_MODE_RWO, + constants.ACCESS_MODE_RWX, + ], + } + + # Create PVCs for CephBlockPool and CephFS + pvc_objects = { + interface: multi_pvc_factory( + interface=interface, + access_modes=modes, + size=size, + num_of_pvc=2, + ) + for interface, modes in access_modes.items() + } + + # Create pods for each interface + self.all_pods = [] + for interface, pvcs in pvc_objects.items(): + pods = create_pods( + pvc_objs=pvcs, + pod_factory=pod_factory, + interface=interface, + pods_for_rwx=2, # Create 2 pods for each RWX PVC + status=constants.STATUS_RUNNING, + ) + assert pods, f"Failed to create pods for {interface}." + log.info(f"Created {len(pods)} pods for interface: {interface}") + self.all_pods.extend(pods) + + # Perform I/O on all pods using ThreadPoolExecutor + log.info("Starting I/O operations on all pods.") + with ThreadPoolExecutor() as executor: + futures = [ + executor.submit( + pod_obj.run_io, storage_type="fs", size="1G", runtime=60 + ) + for pod_obj in self.all_pods + ] + + log.info("Verifying OSD keyrotation for KMS.") + assert self.keyrotation.verify_osd_keyrotation_for_kms( + tries=6, delay=10 + ), "Failed to rotate OSD and NooBaa Keys in KMS." + log.info("Keyrotation verification successful.") + + # Wait for I/O operations to complete + for future in futures: + future.result()