[DPE-3725] Limit secret access and add support for Configuration Hub …

…secret (#77)

pyproject.toml

@@ -42,7 +42,7 @@ ignore_missing_imports = true
 
 [tool.poetry]
 name = "spark8t"
-version = "0.0.5"
+version = "0.0.6"
 description = "This project provides some utilities function and CLI commands to run Spark on K8s."
 authors = [
     "Canonical Data Platform <[email protected]>"

spark8t/cli/params.py

@@ -44,6 +44,21 @@ def add_logging_arguments(parser: ArgumentParser) -> ArgumentParser:
     return parser
 
 
+def add_ignore_configuration_hub(parser: ArgumentParser) -> ArgumentParser:
+    """
+    Add option to exclude the configuration provided by the Spark Configuration Hub
+
+    :param parser: Input parser to decorate with parsing support for logging args.
+    """
+    parser.add_argument(
+        "--ignore-configuration-hub",
+        action="store_true",
+        help="Ignore the configuration provided by Spark Configuration Hub Charm.",
+    )
+
+    return parser
+
+
 def spark_user_parser(parser: ArgumentParser) -> ArgumentParser:
     """
     Add Spark user related argument parsing to the existing parser context

spark8t/cli/pyspark.py

@@ -7,14 +7,15 @@
 
 from spark8t.cli.params import (
     add_config_arguments,
+    add_ignore_configuration_hub,
     add_logging_arguments,
     defaults,
     get_kube_interface,
     k8s_parser,
     parse_arguments_with,
     spark_user_parser,
 )
-from spark8t.domain import ServiceAccount
+from spark8t.domain import PropertyFile, ServiceAccount
 from spark8t.exceptions import AccountNotFound, PrimaryAccountNotFound
 from spark8t.services import K8sServiceAccountRegistry, SparkInterface
 from spark8t.utils import setup_logging
@@ -40,6 +41,9 @@ def main(args: Namespace, logger: Logger):
             args.username
         ) if args.username else PrimaryAccountNotFound()
 
+    if args.ignore_configuration_hub:
+        service_account.configuration_hub_confs = PropertyFile.empty()
+
     SparkInterface(
         service_account=service_account,
         kube_interface=kube_interface,
@@ -49,7 +53,13 @@ def main(args: Namespace, logger: Logger):
 
 if __name__ == "__main__":
     args, extra_args = parse_arguments_with(
-        [add_logging_arguments, k8s_parser, spark_user_parser, add_config_arguments]
+        [
+            add_logging_arguments,
+            k8s_parser,
+            spark_user_parser,
+            add_config_arguments,
+            add_ignore_configuration_hub,
+        ]
     ).parse_known_args()
 
     logger = setup_logging(args.log_level, args.log_conf_file, "spark8t.cli.pyspark")

spark8t/cli/service_account_registry.py

@@ -83,8 +83,11 @@ def create_service_account_registry_parser(parser: ArgumentParser):
     parse_arguments_with(
         [spark_user_parser],
         subparsers.add_parser(Actions.GET_CONFIG.value, parents=[base_parser]),
+    ).add_argument(
+        "--ignore-configuration-hub",
+        action="store_true",
+        help="Boolean to ignore configuration hub generated options.",
     )
-
     #  subparser for sa-conf-del
     parse_arguments_with(
         [spark_user_parser],
@@ -166,6 +169,8 @@ def main(args: Namespace, logger: Logger):
         if maybe_service_account is None:
             raise AccountNotFound(input_service_account.id)
 
+        if args.ignore_configuration_hub:
+            maybe_service_account.configuration_hub_confs = PropertyFile.empty()
         maybe_service_account.configurations.log(print)
 
     elif args.action == Actions.CLEAR_CONFIG:

spark8t/cli/spark_shell.py

@@ -7,14 +7,15 @@
 
 from spark8t.cli.params import (
     add_config_arguments,
+    add_ignore_configuration_hub,
     add_logging_arguments,
     defaults,
     get_kube_interface,
     k8s_parser,
     parse_arguments_with,
     spark_user_parser,
 )
-from spark8t.domain import ServiceAccount
+from spark8t.domain import PropertyFile, ServiceAccount
 from spark8t.exceptions import AccountNotFound, PrimaryAccountNotFound
 from spark8t.services import K8sServiceAccountRegistry, SparkInterface
 from spark8t.utils import setup_logging
@@ -40,6 +41,9 @@ def main(args: Namespace, logger: Logger):
             args.username
         ) if args.username else PrimaryAccountNotFound()
 
+    if args.ignore_configuration_hub:
+        service_account.configuration_hub_confs = PropertyFile.empty()
+
     SparkInterface(
         service_account=service_account,
         kube_interface=kube_interface,
@@ -49,7 +53,13 @@ def main(args: Namespace, logger: Logger):
 
 if __name__ == "__main__":
     args, extra_args = parse_arguments_with(
-        [add_logging_arguments, k8s_parser, spark_user_parser, add_config_arguments]
+        [
+            add_logging_arguments,
+            k8s_parser,
+            spark_user_parser,
+            add_config_arguments,
+            add_ignore_configuration_hub,
+        ]
     ).parse_known_args()
 
     logger = setup_logging(

spark8t/cli/spark_sql.py

@@ -7,14 +7,15 @@
 
 from spark8t.cli.params import (
     add_config_arguments,
+    add_ignore_configuration_hub,
     add_logging_arguments,
     defaults,
     get_kube_interface,
     k8s_parser,
     parse_arguments_with,
     spark_user_parser,
 )
-from spark8t.domain import ServiceAccount
+from spark8t.domain import PropertyFile, ServiceAccount
 from spark8t.exceptions import AccountNotFound, PrimaryAccountNotFound
 from spark8t.services import K8sServiceAccountRegistry, SparkInterface
 from spark8t.utils import setup_logging
@@ -40,6 +41,9 @@ def main(args: Namespace, logger: Logger):
             args.username
         ) if args.username else PrimaryAccountNotFound()
 
+    if args.ignore_configuration_hub:
+        service_account.configuration_hub_confs = PropertyFile.empty()
+
     SparkInterface(
         service_account=service_account,
         kube_interface=kube_interface,
@@ -49,7 +53,13 @@ def main(args: Namespace, logger: Logger):
 
 if __name__ == "__main__":
     args, extra_args = parse_arguments_with(
-        [add_logging_arguments, k8s_parser, spark_user_parser, add_config_arguments]
+        [
+            add_logging_arguments,
+            k8s_parser,
+            spark_user_parser,
+            add_config_arguments,
+            add_ignore_configuration_hub,
+        ]
     ).parse_known_args()
 
     logger = setup_logging(args.log_level, args.log_conf_file, "spark8t.cli.spark_sql")

spark8t/cli/spark_submit.py

@@ -8,14 +8,15 @@
 from spark8t.cli.params import (
     add_config_arguments,
     add_deploy_arguments,
+    add_ignore_configuration_hub,
     add_logging_arguments,
     defaults,
     get_kube_interface,
     k8s_parser,
     parse_arguments_with,
     spark_user_parser,
 )
-from spark8t.domain import ServiceAccount
+from spark8t.domain import PropertyFile, ServiceAccount
 from spark8t.exceptions import AccountNotFound, PrimaryAccountNotFound
 from spark8t.services import K8sServiceAccountRegistry, SparkInterface
 from spark8t.utils import setup_logging
@@ -41,11 +42,19 @@ def main(args: Namespace, logger: Logger):
             args.username
         ) if args.username else PrimaryAccountNotFound()
 
+    if args.ignore_configuration_hub:
+        service_account.configuration_hub_confs = PropertyFile.empty()
+
     SparkInterface(
         service_account=service_account,
         kube_interface=kube_interface,
         defaults=defaults,
-    ).spark_submit(args.deploy_mode, args.conf, args.properties_file, extra_args)
+    ).spark_submit(
+        args.deploy_mode,
+        args.conf,
+        args.properties_file,
+        extra_args,
+    )
 
 
 if __name__ == "__main__":
@@ -56,6 +65,7 @@ def main(args: Namespace, logger: Logger):
             spark_user_parser,
             add_deploy_arguments,
             add_config_arguments,
+            add_ignore_configuration_hub,
         ]
     ).parse_known_args()
 

spark8t/domain.py

@@ -300,6 +300,7 @@ class ServiceAccount:
     api_server: str
     primary: bool = False
     extra_confs: PropertyFile = PropertyFile.empty()
+    configuration_hub_confs: PropertyFile = PropertyFile.empty()
 
     @property
     def id(self):
@@ -318,7 +319,9 @@ def _k8s_configurations(self):
     @property
     def configurations(self) -> PropertyFile:
         """Return the service account configuration, associated to a given spark service account."""
-        return self.extra_confs + self._k8s_configurations
+        return (
+            self.extra_confs + self.configuration_hub_confs + self._k8s_configurations
+        )
 
 
 class KubernetesResourceType(str, Enum):

spark8t/literals.py

@@ -1,3 +1,4 @@
 MANAGED_BY_LABELNAME = "app.kubernetes.io/managed-by"
 PRIMARY_LABELNAME = "app.kubernetes.io/spark8t-primary"
 SPARK8S_LABEL = "spark8t"
+CONFIGURATION_HUB_LABEL = "configuration-hub-conf"

spark8t/resources/templates/role_yaml.tmpl

@@ -13,10 +13,30 @@ rules:
   - configmaps
   - services
   - serviceaccounts
-  - secrets
   verbs:
   - create
   - get
   - list
   - watch
   - delete
+  - deletecollection
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  resourceNames:
+  - spark8t-sa-conf-{{username}}
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  resourceNames:
+  - configuration-hub-conf-{{username}}
+  verbs:
+  - get