From 56de5c25cc3a85aeebdc97fa83400bc736b4a36a Mon Sep 17 00:00:00 2001 From: seolmin Date: Wed, 17 Jan 2024 23:03:32 +0900 Subject: [PATCH] feat: add prowler service for Google Cloud --- .../plugin/connector/aws_prowler_connector.py | 42 +- .../connector/google_prowler_connector.py | 139 ++++++ src/cloudforet/plugin/lib/__init__.py | 1 + src/cloudforet/plugin/lib/profile_manager.py | 57 +++ src/cloudforet/plugin/manager/__init__.py | 1 + .../plugin/manager/google_prowler_manager.py | 407 ++++++++++++++++++ .../plugin/model/prowler/collector.py | 4 +- .../plugin/service/collector_service.py | 2 + 8 files changed, 616 insertions(+), 37 deletions(-) create mode 100644 src/cloudforet/plugin/connector/google_prowler_connector.py create mode 100644 src/cloudforet/plugin/lib/__init__.py create mode 100644 src/cloudforet/plugin/lib/profile_manager.py create mode 100644 src/cloudforet/plugin/manager/google_prowler_manager.py diff --git a/src/cloudforet/plugin/connector/aws_prowler_connector.py b/src/cloudforet/plugin/connector/aws_prowler_connector.py index 83486a8..5be3e85 100644 --- a/src/cloudforet/plugin/connector/aws_prowler_connector.py +++ b/src/cloudforet/plugin/connector/aws_prowler_connector.py @@ -7,6 +7,7 @@ from spaceone.core import utils from spaceone.core.connector import BaseConnector +from cloudforet.plugin.lib import ProfileManager from cloudforet.plugin.error.custom import * from cloudforet.plugin.model.prowler.collector import COMPLIANCE_FRAMEWORKS @@ -19,47 +20,19 @@ _AWS_PROFILE_DIR = _AWS_PROFILE_PATH.rsplit("/", 1)[0] -class AWSProfileManager: - def __init__(self, credentials: dict): - self._profile_name = utils.random_string() - self._source_profile_name = None - self._credentials = credentials - - @property - def profile_name(self) -> str: - return self._profile_name - - @property - def source_profile_name(self) -> str: - return self._source_profile_name - - @source_profile_name.setter - def source_profile_name(self, value: str): - self._source_profile_name = value - - @property - def credentials(self) -> dict: - return self._credentials - - def __enter__(self) -> "AWSProfileManager": - self._add_aws_profile() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self._remove_aws_profile() - - def _add_aws_profile(self): +class AWSProfileManager(ProfileManager): + def _add_profile(self): _LOGGER.debug(f"[_AWSProfileManager] add aws profile: {self._profile_name}") aws_profile = configparser.ConfigParser() if os.path.exists(_AWS_PROFILE_PATH) is False: - self._create_aws_profile_file(aws_profile) + self._create_profile_file(aws_profile) aws_profile.read(_AWS_PROFILE_PATH) if self.profile_name in aws_profile.sections(): - self._remove_aws_profile(aws_profile) + self._remove_profile(aws_profile) aws_profile.add_section(self.profile_name) @@ -104,14 +77,13 @@ def _add_aws_profile(self): with open(_AWS_PROFILE_PATH, "w") as f: aws_profile.write(f) - @staticmethod - def _create_aws_profile_file(aws_profile: configparser.ConfigParser): + def _create_profile_file(self, aws_profile: configparser.ConfigParser): os.makedirs(_AWS_PROFILE_DIR, exist_ok=True) aws_profile["default"] = {} with open(_AWS_PROFILE_PATH, "w") as f: aws_profile.write(f) - def _remove_aws_profile(self, aws_profile: configparser.ConfigParser = None): + def _remove_profile(self, aws_profile: configparser.ConfigParser = None): _LOGGER.debug(f"[_AWSProfileManager] remove aws profile: {self._profile_name}") if aws_profile is None: diff --git a/src/cloudforet/plugin/connector/google_prowler_connector.py b/src/cloudforet/plugin/connector/google_prowler_connector.py new file mode 100644 index 0000000..f3bc35e --- /dev/null +++ b/src/cloudforet/plugin/connector/google_prowler_connector.py @@ -0,0 +1,139 @@ +import os +import json +import logging +import tempfile +import configparser +import subprocess +from typing import List + +from spaceone.core import utils +from spaceone.core.connector import BaseConnector +from cloudforet.plugin.lib import ProfileManager +from cloudforet.plugin.error.custom import * +from cloudforet.plugin.model.prowler.collector import COMPLIANCE_FRAMEWORKS + +__all__ = ["GoogleProwlerConnector"] +REQUIRED_SECRET_KEYS = [ + "type", + "project_id", + "private_key_id", + "private_key", + "client_email", + "client_id", + "auth_uri", + "token_uri", + "auth_provider_x509_cert_url", + "client_x509_cert_url", + "universe_domain", +] + +_LOGGER = logging.getLogger(__name__) +_GOOGLE_CLOUD_PROFILE_PATH = os.environ.get( + "GOOGLE_CLOUD_SHARED_CREDENTIALS_FILE", + os.path.expanduser("~/.google_cloud/credentials"), +) +_GOOGLE_CLOUD_PROFILE_DIR = _GOOGLE_CLOUD_PROFILE_PATH.rsplit("/", 1)[0] + + +class GoogleProfileManager(ProfileManager): + def _add_profile(self): + _LOGGER.debug( + f"[_GoogleProfileManager] add google profile: {self._profile_name}" + ) + json_file_path = os.path.join( + _GOOGLE_CLOUD_PROFILE_DIR, f"{self._profile_name}.json" + ) + + self.source_profile_path = json_file_path + + if os.path.exists(_GOOGLE_CLOUD_PROFILE_PATH) is False: + self._create_profile_file() + else: + if not os.path.exists(json_file_path): + self._create_profile_file() + + def _create_profile_file(self, **kwargs): + os.makedirs(_GOOGLE_CLOUD_PROFILE_DIR, exist_ok=True) + + secret_data_json = json.dumps(self._credentials) + with open(self.source_profile_path, "w") as f: + f.write(secret_data_json) + + def _remove_profile(self, json_file_path=None): + _LOGGER.debug( + f"[_GoogleProfileManager] remove google cloud profile: {self._profile_name}" + ) + + if json_file_path: + os.remove(json_file_path) + + +class GoogleProwlerConnector(BaseConnector): + provider = "google_cloud" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._temp_dir = None + + def verify_client(self, options: dict, secret_data: dict, schema: str): + self._check_secret_data(secret_data) + + with tempfile.TemporaryDirectory(): + with GoogleProfileManager(secret_data) as google_profile: + cmd = self._command_prefix(google_profile.source_profile_path) + cmd += ["-l"] + _LOGGER.debug(f"[verify_client] command: {cmd}") + response = subprocess.run( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE + ) + if response.returncode != 0: + raise ERROR_PROWLER_EXECUTION_FAILED( + reason=response.stderr.decode("utf-8") + ) + + def check(self, options: dict, secret_data: dict, schema: str): + self._check_secret_data(secret_data) + + compliance_framework = COMPLIANCE_FRAMEWORKS["google_cloud"].get( + options["compliance_framework"] + ) + + with tempfile.TemporaryDirectory() as temp_dir: + with GoogleProfileManager(secret_data) as google_profile: + cmd = self._command_prefix(google_profile.source_profile_path) + + cmd += ["-M", "json", "-o", temp_dir, "-F", "output", "-z"] + cmd += ["--compliance", compliance_framework] + + _LOGGER.debug(f"[check] command: {cmd}") + + response = subprocess.run( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE + ) + if response.returncode != 0: + raise ERROR_PROWLER_EXECUTION_FAILED( + reason=response.stderr.decode("utf-8") + ) + + output_json_file = os.path.join(temp_dir, "output.json") + check_results = utils.load_json_from_file(output_json_file) + return check_results + + @staticmethod + def _check_secret_data(secret_data): + missing_keys = [key for key in REQUIRED_SECRET_KEYS if key not in secret_data] + if missing_keys: + for key in missing_keys: + raise ERROR_REQUIRED_PARAMETER(key=f"secret_data.{key}") + + @staticmethod + def _command_prefix(source_profile_path) -> List[str]: + return [ + "python3", + "-m", + "prowler", + "gcp", + "--credentials-file", + source_profile_path, + "-b", + ] diff --git a/src/cloudforet/plugin/lib/__init__.py b/src/cloudforet/plugin/lib/__init__.py new file mode 100644 index 0000000..7879391 --- /dev/null +++ b/src/cloudforet/plugin/lib/__init__.py @@ -0,0 +1 @@ +from cloudforet.plugin.lib.profile_manager import ProfileManager diff --git a/src/cloudforet/plugin/lib/profile_manager.py b/src/cloudforet/plugin/lib/profile_manager.py new file mode 100644 index 0000000..864bd1d --- /dev/null +++ b/src/cloudforet/plugin/lib/profile_manager.py @@ -0,0 +1,57 @@ +import logging +import abc + +from spaceone.core import utils + +_LOGGER = logging.getLogger(__name__) + + +class ProfileManager: + def __init__(self, credentials: dict): + self._profile_name = utils.random_string() + self._source_profile_name = None + self._source_profile_path = None + self._credentials = credentials + + @property + def profile_name(self) -> str: + return self._profile_name + + @property + def source_profile_name(self) -> str: + return self._source_profile_name + + @source_profile_name.setter + def source_profile_name(self, value: str): + self._source_profile_name = value + + @property + def source_profile_path(self) -> str: + return self._source_profile_path + + @source_profile_path.setter + def source_profile_path(self, value: str): + self._source_profile_path = value + + @property + def credentials(self) -> dict: + return self._credentials + + def __enter__(self) -> "ProfileManager": + self._add_profile() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._remove_profile() + + @abc.abstractmethod + def _add_profile(self): + raise NotImplemented("Please implement _add_profile method") + + @abc.abstractmethod + def _create_profile_file(self, profile): + raise NotImplemented("Please implement _create_profile_file method") + + @abc.abstractmethod + def _remove_profile(self, profile=None): + raise NotImplemented("Please implement _remove_profile method") diff --git a/src/cloudforet/plugin/manager/__init__.py b/src/cloudforet/plugin/manager/__init__.py index aa872b1..e2bc302 100644 --- a/src/cloudforet/plugin/manager/__init__.py +++ b/src/cloudforet/plugin/manager/__init__.py @@ -1,2 +1,3 @@ from cloudforet.plugin.manager.aws_prowler_manager import AWSProwlerManager from cloudforet.plugin.manager.azure_prowler_manager import AzureProwlerManager +from cloudforet.plugin.manager.google_prowler_manager import GoogleProwlerManager diff --git a/src/cloudforet/plugin/manager/google_prowler_manager.py b/src/cloudforet/plugin/manager/google_prowler_manager.py new file mode 100644 index 0000000..656a803 --- /dev/null +++ b/src/cloudforet/plugin/manager/google_prowler_manager.py @@ -0,0 +1,407 @@ +import time +import random +import logging +from typing import Generator, List +from prowler.lib.check.check import bulk_load_compliance_frameworks + +from cloudforet.plugin.connector.google_prowler_connector import GoogleProwlerConnector +from cloudforet.plugin.error.custom import * +from cloudforet.plugin.manager.collector_manager import CollectorManager +from cloudforet.plugin.model.prowler.cloud_service_type import CloudServiceType +from cloudforet.plugin.model.prowler.collector import COMPLIANCE_FRAMEWORKS + +_LOGGER = logging.getLogger(__name__) + +_SEVERITY_MAP = { + "critical": "CRITICAL", + "high": "HIGH", + "medium": "MEDIUM", + "low": "LOW", + "informational": "INFORMATIONAL", +} + +_SEVERITY_SCORE_MAP = { + "CRITICAL": 4, + "HIGH": 3, + "MEDIUM": 2, + "LOW": 1, + "INFORMATIONAL": 0, + "UNKNOWN": 1, +} + + +class GoogleProwlerManager(CollectorManager): + provider = "google_cloud" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.google_prowler_connector: GoogleProwlerConnector = ( + self.locator.get_connector(GoogleProwlerConnector) + ) + self.provider = "google_cloud" + self.cloud_service_group = "Prowler" + self.cloud_service_type = None + self.compliance_framework_info = {} + + def collect( + self, options: dict, secret_data: dict, schema: str + ) -> Generator[dict, None, None]: + self.cloud_service_type = options["compliance_framework"] + self._check_compliance_framework() + self._load_compliance_framework_info() + + self._wait_random_time() + + try: + check_results = self.google_prowler_connector.check( + options, secret_data, schema + ) + + # Return Cloud Service Type + cloud_service_type = CloudServiceType( + name=self.cloud_service_type, provider=self.provider + ) + cloud_service_type.metadata["query_sets"][0][ + "name" + ] = f"Google Cloud {self.cloud_service_type}" + yield self.make_response( + cloud_service_type.dict(), + {"1": ["name", "group", "provider"]}, + resource_type="inventory.CloudServiceType", + ) + + # Return compliance results (Cloud Services) + for compliance_result in self.make_compliance_results(check_results): + yield self.make_response( + compliance_result, + { + "1": [ + "reference.resource_id", + "provider", + "cloud_service_type", + "cloud_service_group", + "account", + ] + }, + ) + + except Exception as e: + yield self.error_response(e) + + def make_compliance_results(self, check_results: List[dict]) -> List[dict]: + compliance_results = {} + for check_result in check_results: + account = check_result["ProjectId"] + requirements = check_result.get("Compliance", {}).get( + self.cloud_service_type, [] + ) + for requirement_id in requirements: + compliance_id = f"prowler:google_cloud:{account}:{self.cloud_service_type}:{requirement_id}".lower() + check_id = check_result["CheckID"] + status = check_result["Status"] + region_code = check_result["Location"] + severity = _SEVERITY_MAP.get(check_result["Severity"], "UNKNOWN") + score = _SEVERITY_SCORE_MAP[severity] + + if compliance_id not in compliance_results: + compliance_results[ + compliance_id + ] = self._make_base_compliance_result( + compliance_id, requirement_id, severity, check_result + ) + + check_exists = ( + check_id in compliance_results[compliance_id]["data"]["checks"] + ) + + if compliance_results[compliance_id]["region_code"] != region_code: + compliance_results[compliance_id]["region_code"] = "global" + + compliance_results[compliance_id]["data"][ + "severity" + ] = self._update_severity( + compliance_results[compliance_id]["data"]["severity"], severity + ) + + compliance_results[compliance_id][ + "data" + ] = self._update_compliance_status_and_stats( + compliance_results[compliance_id]["data"], status, score + ) + + compliance_results[compliance_id]["data"]["findings"].append( + self._make_finding(check_result) + ) + + if not check_exists: + compliance_results[compliance_id]["data"]["checks"][ + check_id + ] = self._make_check(check_result) + + compliance_results[compliance_id]["data"]["checks"][ + check_id + ] = self._update_check_status_and_stats( + compliance_results[compliance_id]["data"]["checks"][check_id], + status, + score, + ) + + return self._convert_results(compliance_results) + + def _convert_results(self, compliance_results): + results = [] + for compliance_result in compliance_results.values(): + total_check_count = 0 + pass_check_count = 0 + fail_check_count = 0 + info_check_count = 0 + + compliance_result["data"]["stats"]["score"][ + "percent" + ] = self._calculate_score(compliance_result["data"]["stats"]) + + changed_checks = [] + for check in compliance_result["data"]["checks"].values(): + total_check_count += 1 + if check["status"] == "FAIL": + fail_check_count += 1 + elif check["status"] == "INFO": + info_check_count += 1 + else: + pass_check_count += 1 + + check["display"] = self._make_check_display(check["stats"]) + check["stats"]["score"]["percent"] = self._calculate_score( + check["stats"] + ) + changed_checks.append(check) + + compliance_result["data"]["checks"] = changed_checks + compliance_result["data"]["stats"]["checks"] = { + "total": total_check_count, + "pass": pass_check_count, + "fail": fail_check_count, + "info": info_check_count, + } + + compliance_result["data"]["display"] = self._make_compliance_display( + compliance_result["data"]["stats"] + ) + + results.append(compliance_result) + + return results + + @staticmethod + def _make_check_display(check_stats): + findings_pass = check_stats["findings"]["pass"] + findings_total = check_stats["findings"]["total"] + + return {"findings": f"{findings_pass}/{findings_total}"} + + @staticmethod + def _calculate_score(stats): + score_pass = stats["score"]["pass"] + score_fail = stats["score"]["fail"] + score_total = score_pass + score_fail + + if score_total == 0: + return 0 + else: + return round(score_pass / score_total * 100, 1) + + @staticmethod + def _make_compliance_display(compliance_data_stats): + checks_pass = compliance_data_stats["checks"]["pass"] + checks_total = compliance_data_stats["checks"]["total"] + findings_pass = compliance_data_stats["findings"]["pass"] + findings_total = compliance_data_stats["findings"]["total"] + + return { + "checks": f"{checks_pass}/{checks_total}", + "findings": f"{findings_pass}/{findings_total}", + } + + @staticmethod + def _update_severity(old_severity: str, new_severity: str) -> str: + if _SEVERITY_SCORE_MAP[old_severity] < _SEVERITY_SCORE_MAP[new_severity]: + return new_severity + return old_severity + + def _make_check(self, check_result: dict) -> dict: + check = { + "check_id": check_result["CheckID"], + "check_title": check_result["CheckTitle"], + "service": check_result["ServiceName"], + "sub_service": check_result["SubServiceName"], + "check_type": check_result["CheckType"], + "status": "PASS", + "severity": _SEVERITY_MAP.get(check_result["Severity"], "UNKNOWN"), + "risk": check_result["Risk"], + "remediation": self._make_remediation(check_result["Remediation"]), + "stats": { + "score": {"pass": 0, "fail": 0, "percent": 0}, + "findings": { + "total": 0, + "pass": 0, + "fail": 0, + "info": 0, + }, + }, + } + + return check + + @staticmethod + def _make_remediation(remediation_info): + recommendation = remediation_info.get("Recommendation", {}) + return { + "description": recommendation.get("Text", ""), + "link": recommendation.get("Url", ""), + } + + @staticmethod + def _make_finding(check_result: dict) -> dict: + return { + "finding_id": check_result["FindingUniqueId"], + "check_id": check_result["CheckID"], + "check_title": check_result["CheckTitle"], + "status": check_result["Status"], + "status_extended": check_result["StatusExtended"], + "resource": check_result["ResourceId"] or check_result["ResourceName"], + "resource_type": check_result["ResourceType"], + "region_code": check_result["Location"], + } + + @staticmethod + def _update_check_status_and_stats(check: dict, status: str, score: int) -> dict: + check["stats"]["findings"]["total"] += 1 + + if status == "FAIL": + check["status"] = "FAIL" + check["stats"]["score"]["fail"] += score + check["stats"]["findings"]["fail"] += 1 + elif status == "INFO": + if check["status"] != "FAIL": + check["status"] = "INFO" + check["stats"]["findings"]["info"] += 1 + else: + check["stats"]["score"]["pass"] += score + check["stats"]["findings"]["pass"] += 1 + + return check + + @staticmethod + def _update_compliance_status_and_stats( + compliance_result_data: dict, status: str, score: int + ) -> dict: + compliance_result_data["stats"]["findings"]["total"] += 1 + + if status == "FAIL": + compliance_result_data["status"] = "FAIL" + compliance_result_data["stats"]["score"]["fail"] += score + compliance_result_data["stats"]["findings"]["fail"] += 1 + elif status == "INFO": + if compliance_result_data["status"] != "FAIL": + compliance_result_data["status"] = "INFO" + + compliance_result_data["stats"]["findings"]["info"] += 1 + else: + compliance_result_data["stats"]["score"]["pass"] += score + compliance_result_data["stats"]["findings"]["pass"] += 1 + + # if not check_exists: + # compliance_result_data['stats']['checks']['total'] += 1 + # if status == 'FAIL': + # compliance_result_data['stats']['checks']['fail'] += 1 + # elif status == 'PASS': + # compliance_result_data['stats']['checks']['pass'] += 1 + # else: + # compliance_result_data['stats']['checks']['info'] += 1 + + return compliance_result_data + + def _make_base_compliance_result( + self, compliance_id: str, requirement_id: str, severity: str, check_result: dict + ) -> dict: + compliance_result = { + "name": self.compliance_framework_info[requirement_id], + "reference": { + "resource_id": compliance_id, + }, + "data": { + "requirement_id": requirement_id, + "description": check_result["Description"], + "status": "PASS", + "severity": severity, + "service": check_result["ServiceName"], + "checks": {}, + "findings": [], + "display": { + "score": "", + "score_percent": "", + "checks": "", + "findings": "", + }, + "stats": { + "score": {"pass": 0, "fail": 0, "percent": 0}, + "checks": { + "total": 0, + "pass": 0, + "fail": 0, + "info": 0, + }, + "findings": { + "total": 0, + "pass": 0, + "fail": 0, + "info": 0, + }, + }, + }, + "metadata": { + "view": { + "sub_data": { + "reference": { + "resource_type": "inventory.CloudServiceType", + "options": { + "provider": self.provider, + "cloud_service_group": self.cloud_service_group, + "cloud_service_type": self.cloud_service_type, + }, + } + } + } + }, + "account": check_result["ProjectId"], + "provider": self.provider, + "cloud_service_group": self.cloud_service_group, + "cloud_service_type": self.cloud_service_type, + "region_code": check_result["Location"], + } + + return compliance_result + + @staticmethod + def _wait_random_time(): + random_time = round(random.uniform(0, 5), 2) + _LOGGER.debug(f"[_wait_random_time] sleep time: {random_time}") + + time.sleep(random_time) + + def _check_compliance_framework(self): + all_compliance_frameworks = list(COMPLIANCE_FRAMEWORKS["google_cloud"].keys()) + if self.cloud_service_type not in all_compliance_frameworks: + raise ERROR_INVALID_PARAMETER( + key="options.compliance_framework", + reason=f"Not supported compliance framework. " + f"(compliance_frameworks = {all_compliance_frameworks})", + ) + + def _load_compliance_framework_info(self): + compliance_framework = COMPLIANCE_FRAMEWORKS["google_cloud"][ + self.cloud_service_type + ] + compliance_frameworks = bulk_load_compliance_frameworks("gcp") + for requirement in compliance_frameworks[compliance_framework].Requirements: + self.compliance_framework_info[requirement.Id] = requirement.Description diff --git a/src/cloudforet/plugin/model/prowler/collector.py b/src/cloudforet/plugin/model/prowler/collector.py index 01a59f5..fac71e8 100644 --- a/src/cloudforet/plugin/model/prowler/collector.py +++ b/src/cloudforet/plugin/model/prowler/collector.py @@ -38,7 +38,7 @@ "FedRAMP-Low-Revision-4": "fedramp_low_revision_4_aws", }, "google_cloud": { - "Google-Cloud-Standard": "", + "CIS-2.0": "cis_2.0_gcp", }, "azure": { "Azure-Standard": "", @@ -228,7 +228,7 @@ class GoogleCloudPluginInfo(PluginInfo): "title": "Compliance Framework", "type": "string", "enum": list(COMPLIANCE_FRAMEWORKS["google_cloud"].keys()), - "default": "Google-Cloud-Standard", + "default": "CIS-2.0", }, # 'services': { # 'title': 'Service', diff --git a/src/cloudforet/plugin/service/collector_service.py b/src/cloudforet/plugin/service/collector_service.py index 11a5c5a..dccf053 100644 --- a/src/cloudforet/plugin/service/collector_service.py +++ b/src/cloudforet/plugin/service/collector_service.py @@ -11,6 +11,8 @@ class CollectorService(BaseService): + resource = "Collector" + @transaction @check_required(["options", "options.provider"]) def init(self, params):