diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3c20ba66..ba445dcd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -53,7 +53,7 @@ jobs: integration-test: strategy: - max-parallel: 1 + max-parallel: 2 fail-fast: false matrix: tox-environments: diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py index 714eace4..5cb309b1 100644 --- a/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -295,12 +295,23 @@ def _on_topic_requested(self, event: TopicRequestedEvent): import json import logging from abc import ABC, abstractmethod -from collections import namedtuple +from collections import UserDict, namedtuple from datetime import datetime from enum import Enum -from typing import Callable, Dict, List, Optional, Set, Tuple, Union +from typing import ( + Callable, + Dict, + ItemsView, + KeysView, + List, + Optional, + Set, + Tuple, + Union, + ValuesView, +) -from ops import JujuVersion, Secret, SecretInfo, SecretNotFoundError +from ops import JujuVersion, Model, Secret, SecretInfo, SecretNotFoundError from ops.charm import ( CharmBase, CharmEvents, @@ -320,7 +331,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 24 +LIBPATCH = 35 PYDEPS = ["ops>=2.0.0"] @@ -337,31 +348,46 @@ def _on_topic_requested(self, event: TopicRequestedEvent): PROV_SECRET_PREFIX = "secret-" REQ_SECRET_FIELDS = "requested-secrets" +GROUP_MAPPING_FIELD = "secret_group_mapping" +GROUP_SEPARATOR = "@" + + +class SecretGroup(str): + """Secret groups specific type.""" + +class SecretGroupsAggregate(str): + """Secret groups with option to extend with additional constants.""" -class SecretGroup(Enum): - """Secret groups as constants.""" + def __init__(self): + self.USER = SecretGroup("user") + self.TLS = SecretGroup("tls") + self.EXTRA = SecretGroup("extra") - USER = "user" - TLS = "tls" - EXTRA = "extra" + def __setattr__(self, name, value): + """Setting internal constants.""" + if name in self.__dict__: + raise RuntimeError("Can't set constant!") + else: + super().__setattr__(name, SecretGroup(value)) + + def groups(self) -> list: + """Return the list of stored SecretGroups.""" + return list(self.__dict__.values()) + def get_group(self, group: str) -> Optional[SecretGroup]: + """If the input str translates to a group name, return that.""" + return SecretGroup(group) if group in self.groups() else None -# Local map to associate mappings with secrets potentially as a group -SECRET_LABEL_MAP = { - "username": SecretGroup.USER, - "password": SecretGroup.USER, - "uris": SecretGroup.USER, - "tls": SecretGroup.TLS, - "tls-ca": SecretGroup.TLS, -} + +SECRET_GROUPS = SecretGroupsAggregate() class DataInterfacesError(Exception): """Common ancestor for DataInterfaces related exceptions.""" -class SecretError(Exception): +class SecretError(DataInterfacesError): """Common ancestor for Secrets related exceptions.""" @@ -377,6 +403,10 @@ class SecretsIllegalUpdateError(SecretError): """Secrets aren't yet available for Juju version used.""" +class IllegalOperationError(DataInterfacesError): + """To be used when an operation is not allowed to be performed.""" + + def get_encoded_dict( relation: Relation, member: Union[Unit, Application], field: str ) -> Optional[Dict[str, str]]: @@ -407,7 +437,7 @@ def set_encoded_field( relation.data[member].update({field: json.dumps(value)}) -def diff(event: RelationChangedEvent, bucket: Union[Unit, Application]) -> Diff: +def diff(event: RelationChangedEvent, bucket: Optional[Union[Unit, Application]]) -> Diff: """Retrieves the diff of the data in the relation changed databag. Args: @@ -419,6 +449,9 @@ def diff(event: RelationChangedEvent, bucket: Union[Unit, Application]) -> Diff: keys from the event relation databag. """ # Retrieve the old data from the data key in the application relation databag. + if not bucket: + return Diff([], [], []) + old_data = get_encoded_dict(event.relation, bucket, "data") if not old_data: @@ -432,15 +465,15 @@ def diff(event: RelationChangedEvent, bucket: Union[Unit, Application]) -> Diff: ) # These are the keys that were added to the databag and triggered this event. - added = new_data.keys() - old_data.keys() # pyright: ignore [reportGeneralTypeIssues] + added = new_data.keys() - old_data.keys() # pyright: ignore [reportAssignmentType] # These are the keys that were removed from the databag and triggered this event. - deleted = old_data.keys() - new_data.keys() # pyright: ignore [reportGeneralTypeIssues] + deleted = old_data.keys() - new_data.keys() # pyright: ignore [reportAssignmentType] # These are the keys that already existed in the databag, # but had their values changed. changed = { key - for key in old_data.keys() & new_data.keys() # pyright: ignore [reportGeneralTypeIssues] - if old_data[key] != new_data[key] # pyright: ignore [reportGeneralTypeIssues] + for key in old_data.keys() & new_data.keys() # pyright: ignore [reportAssignmentType] + if old_data[key] != new_data[key] # pyright: ignore [reportAssignmentType] } # Convert the new_data to a serializable format and save it for a next diff check. set_encoded_field(event.relation, bucket, "data", new_data) @@ -453,13 +486,14 @@ def leader_only(f): """Decorator to ensure that only leader can perform given operation.""" def wrapper(self, *args, **kwargs): - if not self.local_unit.is_leader(): + if self.component == self.local_app and not self.local_unit.is_leader(): logger.error( "This operation (%s()) can only be performed by the leader unit", f.__name__ ) return return f(self, *args, **kwargs) + wrapper.leader_only = True return wrapper @@ -474,6 +508,34 @@ def wrapper(self, *args, **kwargs): return wrapper +def dynamic_secrets_only(f): + """Decorator to ensure that certain operations would be only executed when NO static secrets are defined.""" + + def wrapper(self, *args, **kwargs): + if self.static_secret_fields: + raise IllegalOperationError( + "Unsafe usage of statically and dynamically defined secrets, aborting." + ) + return f(self, *args, **kwargs) + + return wrapper + + +def either_static_or_dynamic_secrets(f): + """Decorator to ensure that static and dynamic secrets won't be used in parallel.""" + + def wrapper(self, *args, **kwargs): + if self.static_secret_fields and set(self.current_secret_fields) - set( + self.static_secret_fields + ): + raise IllegalOperationError( + "Unsafe usage of statically and dynamically defined secrets, aborting." + ) + return f(self, *args, **kwargs) + + return wrapper + + class Scope(Enum): """Peer relations scope.""" @@ -481,28 +543,52 @@ class Scope(Enum): UNIT = "unit" +################################################################################ +# Secrets internal caching +################################################################################ + + class CachedSecret: """Locally cache a secret. The data structure is precisely re-using/simulating as in the actual Secret Storage """ - def __init__(self, charm: CharmBase, label: str, secret_uri: Optional[str] = None): + def __init__( + self, + model: Model, + component: Union[Application, Unit], + label: str, + secret_uri: Optional[str] = None, + legacy_labels: List[str] = [], + ): self._secret_meta = None self._secret_content = {} self._secret_uri = secret_uri self.label = label - self.charm = charm + self._model = model + self.component = component + self.legacy_labels = legacy_labels + self.current_label = None - def add_secret(self, content: Dict[str, str], relation: Relation) -> Secret: + def add_secret( + self, + content: Dict[str, str], + relation: Optional[Relation] = None, + label: Optional[str] = None, + ) -> Secret: """Create a new secret.""" if self._secret_uri: raise SecretAlreadyExistsError( "Secret is already defined with uri %s", self._secret_uri ) - secret = self.charm.app.add_secret(content, label=self.label) - secret.grant(relation) + label = self.label if not label else label + + secret = self.component.add_secret(content, label=label) + if relation and relation.app != self._model.app: + # If it's not a peer relation, grant is to be applied + secret.grant(relation) self._secret_uri = secret.id self._secret_meta = secret return self._secret_meta @@ -513,13 +599,20 @@ def meta(self) -> Optional[Secret]: if not self._secret_meta: if not (self._secret_uri or self.label): return - try: - self._secret_meta = self.charm.model.get_secret(label=self.label) - except SecretNotFoundError: - if self._secret_uri: - self._secret_meta = self.charm.model.get_secret( - id=self._secret_uri, label=self.label - ) + + for label in [self.label] + self.legacy_labels: + try: + self._secret_meta = self._model.get_secret(label=label) + except SecretNotFoundError: + pass + else: + if label != self.label: + self.current_label = label + break + + # If still not found, to be checked by URI, to be labelled with the proposed label + if not self._secret_meta and self._secret_uri: + self._secret_meta = self._model.get_secret(id=self._secret_uri, label=self.label) return self._secret_meta def get_content(self) -> Dict[str, str]: @@ -531,19 +624,42 @@ def get_content(self) -> Dict[str, str]: except (ValueError, ModelError) as err: # https://bugs.launchpad.net/juju/+bug/2042596 # Only triggered when 'refresh' is set - msg = "ERROR either URI or label should be used for getting an owned secret but not both" - if isinstance(err, ModelError) and msg not in str(err): + known_model_errors = [ + "ERROR either URI or label should be used for getting an owned secret but not both", + "ERROR secret owner cannot use --refresh", + ] + if isinstance(err, ModelError) and not any( + msg in str(err) for msg in known_model_errors + ): raise # Due to: ValueError: Secret owner cannot use refresh=True self._secret_content = self.meta.get_content() return self._secret_content + def _move_to_new_label_if_needed(self): + """Helper function to re-create the secret with a different label.""" + if not self.current_label or not (self.meta and self._secret_meta): + return + + # Create a new secret with the new label + old_meta = self._secret_meta + content = self._secret_meta.get_content() + + # I wish we could just check if we are the owners of the secret... + try: + self._secret_meta = self.add_secret(content, label=self.label) + except ModelError as err: + if "this unit is not the leader" not in str(err): + raise + old_meta.remove_all_revisions() + def set_content(self, content: Dict[str, str]) -> None: """Setting cached secret content.""" if not self.meta: return if content: + self._move_to_new_label_if_needed() self.meta.set_content(content) self._secret_content = content else: @@ -554,18 +670,35 @@ def get_info(self) -> Optional[SecretInfo]: if self.meta: return self.meta.get_info() + def remove(self) -> None: + """Remove secret.""" + if not self.meta: + raise SecretsUnavailableError("Non-existent secret was attempted to be removed.") + try: + self.meta.remove_all_revisions() + except SecretNotFoundError: + pass + self._secret_content = {} + self._secret_meta = None + self._secret_uri = None + class SecretCache: """A data structure storing CachedSecret objects.""" - def __init__(self, charm): - self.charm = charm + def __init__(self, model: Model, component: Union[Application, Unit]): + self._model = model + self.component = component self._secrets: Dict[str, CachedSecret] = {} - def get(self, label: str, uri: Optional[str] = None) -> Optional[CachedSecret]: + def get( + self, label: str, uri: Optional[str] = None, legacy_labels: List[str] = [] + ) -> Optional[CachedSecret]: """Getting a secret from Juju Secret store or cache.""" if not self._secrets.get(label): - secret = CachedSecret(self.charm, label, uri) + secret = CachedSecret( + self._model, self.component, label, uri, legacy_labels=legacy_labels + ) if secret.meta: self._secrets[label] = secret return self._secrets.get(label) @@ -575,37 +708,172 @@ def add(self, label: str, content: Dict[str, str], relation: Relation) -> Cached if self._secrets.get(label): raise SecretAlreadyExistsError(f"Secret {label} already exists") - secret = CachedSecret(self.charm, label) + secret = CachedSecret(self._model, self.component, label) secret.add_secret(content, relation) self._secrets[label] = secret return self._secrets[label] + def remove(self, label: str) -> None: + """Remove a secret from the cache.""" + if secret := self.get(label): + try: + secret.remove() + self._secrets.pop(label) + except (SecretsUnavailableError, KeyError): + pass + else: + return + logging.debug("Non-existing Juju Secret was attempted to be removed %s", label) + + +################################################################################ +# Relation Data base/abstract ancestors (i.e. parent classes) +################################################################################ + + +# Base Data + -# Base DataRelation +class DataDict(UserDict): + """Python Standard Library 'dict' - like representation of Relation Data.""" + def __init__(self, relation_data: "Data", relation_id: int): + self.relation_data = relation_data + self.relation_id = relation_id -class DataRelation(Object, ABC): + @property + def data(self) -> Dict[str, str]: + """Return the full content of the Abstract Relation Data dictionary.""" + result = self.relation_data.fetch_my_relation_data([self.relation_id]) + try: + result_remote = self.relation_data.fetch_relation_data([self.relation_id]) + except NotImplementedError: + result_remote = {self.relation_id: {}} + if result: + result_remote[self.relation_id].update(result[self.relation_id]) + return result_remote.get(self.relation_id, {}) + + def __setitem__(self, key: str, item: str) -> None: + """Set an item of the Abstract Relation Data dictionary.""" + self.relation_data.update_relation_data(self.relation_id, {key: item}) + + def __getitem__(self, key: str) -> str: + """Get an item of the Abstract Relation Data dictionary.""" + result = None + + # Avoiding "leader_only" error when cross-charm non-leader unit, not to report useless error + if ( + not hasattr(self.relation_data.fetch_my_relation_field, "leader_only") + or self.relation_data.component != self.relation_data.local_app + or self.relation_data.local_unit.is_leader() + ): + result = self.relation_data.fetch_my_relation_field(self.relation_id, key) + + if not result: + try: + result = self.relation_data.fetch_relation_field(self.relation_id, key) + except NotImplementedError: + pass + + if not result: + raise KeyError + return result + + def __eq__(self, d: dict) -> bool: + """Equality.""" + return self.data == d + + def __repr__(self) -> str: + """String representation Abstract Relation Data dictionary.""" + return repr(self.data) + + def __len__(self) -> int: + """Length of the Abstract Relation Data dictionary.""" + return len(self.data) + + def __delitem__(self, key: str) -> None: + """Delete an item of the Abstract Relation Data dictionary.""" + self.relation_data.delete_relation_data(self.relation_id, [key]) + + def has_key(self, key: str) -> bool: + """Does the key exist in the Abstract Relation Data dictionary?""" + return key in self.data + + def update(self, items: Dict[str, str]): + """Update the Abstract Relation Data dictionary.""" + self.relation_data.update_relation_data(self.relation_id, items) + + def keys(self) -> KeysView[str]: + """Keys of the Abstract Relation Data dictionary.""" + return self.data.keys() + + def values(self) -> ValuesView[str]: + """Values of the Abstract Relation Data dictionary.""" + return self.data.values() + + def items(self) -> ItemsView[str, str]: + """Items of the Abstract Relation Data dictionary.""" + return self.data.items() + + def pop(self, item: str) -> str: + """Pop an item of the Abstract Relation Data dictionary.""" + result = self.relation_data.fetch_my_relation_field(self.relation_id, item) + if not result: + raise KeyError(f"Item {item} doesn't exist.") + self.relation_data.delete_relation_data(self.relation_id, [item]) + return result + + def __contains__(self, item: str) -> bool: + """Does the Abstract Relation Data dictionary contain item?""" + return item in self.data.values() + + def __iter__(self): + """Iterate through the Abstract Relation Data dictionary.""" + return iter(self.data) + + def get(self, key: str, default: Optional[str] = None) -> Optional[str]: + """Safely get an item of the Abstract Relation Data dictionary.""" + try: + if result := self[key]: + return result + except KeyError: + return default + + +class Data(ABC): """Base relation data mainpulation (abstract) class.""" - def __init__(self, charm: CharmBase, relation_name: str) -> None: - super().__init__(charm, relation_name) - self.charm = charm - self.local_app = self.charm.model.app - self.local_unit = self.charm.unit + SCOPE = Scope.APP + + # Local map to associate mappings with secrets potentially as a group + SECRET_LABEL_MAP = { + "username": SECRET_GROUPS.USER, + "password": SECRET_GROUPS.USER, + "uris": SECRET_GROUPS.USER, + "tls": SECRET_GROUPS.TLS, + "tls-ca": SECRET_GROUPS.TLS, + } + + def __init__( + self, + model: Model, + relation_name: str, + ) -> None: + self._model = model + self.local_app = self._model.app + self.local_unit = self._model.unit self.relation_name = relation_name - self.framework.observe( - charm.on[relation_name].relation_changed, - self._on_relation_changed_event, - ) self._jujuversion = None - self.secrets = SecretCache(self.charm) + self.component = self.local_app if self.SCOPE == Scope.APP else self.local_unit + self.secrets = SecretCache(self._model, self.component) + self.data_component = None @property def relations(self) -> List[Relation]: """The list of Relation instances associated with this relation_name.""" return [ relation - for relation in self.charm.model.relations[self.relation_name] + for relation in self._model.relations[self.relation_name] if self._is_relation_active(relation) ] @@ -616,12 +884,12 @@ def secrets_enabled(self): self._jujuversion = JujuVersion.from_environ() return self._jujuversion.has_secrets - # Mandatory overrides for internal/helper methods + @property + def secret_label_map(self): + """Exposing secret-label map via a property -- could be overridden in descendants!""" + return self.SECRET_LABEL_MAP - @abstractmethod - def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: - """Event emitted when the relation data has changed.""" - raise NotImplementedError + # Mandatory overrides for internal/helper methods @abstractmethod def _get_relation_secret( @@ -675,12 +943,11 @@ def _generate_secret_label( relation_name: str, relation_id: int, group_mapping: SecretGroup ) -> str: """Generate unique group_mappings for secrets within a relation context.""" - return f"{relation_name}.{relation_id}.{group_mapping.value}.secret" + return f"{relation_name}.{relation_id}.{group_mapping}.secret" - @staticmethod - def _generate_secret_field_name(group_mapping: SecretGroup) -> str: + def _generate_secret_field_name(self, group_mapping: SecretGroup) -> str: """Generate unique group_mappings for secrets within a relation context.""" - return f"{PROV_SECRET_PREFIX}{group_mapping.value}" + return f"{PROV_SECRET_PREFIX}{group_mapping}" def _relation_from_secret_label(self, secret_label: str) -> Optional[Relation]: """Retrieve the relation that belongs to a secret label.""" @@ -705,8 +972,7 @@ def _relation_from_secret_label(self, secret_label: str) -> Optional[Relation]: except ModelError: return - @staticmethod - def _group_secret_fields(secret_fields: List[str]) -> Dict[SecretGroup, List[str]]: + def _group_secret_fields(self, secret_fields: List[str]) -> Dict[SecretGroup, List[str]]: """Helper function to arrange secret mappings under their group. NOTE: All unrecognized items end up in the 'extra' secret bucket. @@ -714,44 +980,42 @@ def _group_secret_fields(secret_fields: List[str]) -> Dict[SecretGroup, List[str """ secret_fieldnames_grouped = {} for key in secret_fields: - if group := SECRET_LABEL_MAP.get(key): + if group := self.secret_label_map.get(key): secret_fieldnames_grouped.setdefault(group, []).append(key) else: - secret_fieldnames_grouped.setdefault(SecretGroup.EXTRA, []).append(key) + secret_fieldnames_grouped.setdefault(SECRET_GROUPS.EXTRA, []).append(key) return secret_fieldnames_grouped def _get_group_secret_contents( self, relation: Relation, group: SecretGroup, - secret_fields: Optional[Union[Set[str], List[str]]] = None, + secret_fields: Union[Set[str], List[str]] = [], ) -> Dict[str, str]: """Helper function to retrieve collective, requested contents of a secret.""" - if not secret_fields: - secret_fields = [] - if (secret := self._get_relation_secret(relation.id, group)) and ( secret_data := secret.get_content() ): - return {k: v for k, v in secret_data.items() if k in secret_fields} + return { + k: v for k, v in secret_data.items() if not secret_fields or k in secret_fields + } return {} - @staticmethod def _content_for_secret_group( - content: Dict[str, str], secret_fields: Set[str], group_mapping: SecretGroup + self, content: Dict[str, str], secret_fields: Set[str], group_mapping: SecretGroup ) -> Dict[str, str]: """Select : pairs from input, that belong to this particular Secret group.""" - if group_mapping == SecretGroup.EXTRA: + if group_mapping == SECRET_GROUPS.EXTRA: return { k: v for k, v in content.items() - if k in secret_fields and k not in SECRET_LABEL_MAP.keys() + if k in secret_fields and k not in self.secret_label_map.keys() } return { k: v for k, v in content.items() - if k in secret_fields and SECRET_LABEL_MAP.get(k) == group_mapping + if k in secret_fields and self.secret_label_map.get(k) == group_mapping } @juju_secrets_only @@ -780,11 +1044,11 @@ def _process_secret_fields( # If the relation started on a databag, we just stay on the databag # (Rolling upgrades may result in a relation starting on databag, getting secrets enabled on-the-fly) - # self.local_app is sufficient to check (ignored if Requires, never has secrets -- works if Provides) + # self.local_app is sufficient to check (ignored if Requires, never has secrets -- works if Provider) fallback_to_databag = ( req_secret_fields - and self.local_unit.is_leader() - and set(req_secret_fields) & set(relation.data[self.local_app]) + and (self.local_unit == self._model.unit and self.local_unit.is_leader()) + and set(req_secret_fields) & set(relation.data[self.component]) ) normal_fields = set(impacted_rel_fields) @@ -807,26 +1071,28 @@ def _process_secret_fields( return (result, normal_fields) def _fetch_relation_data_without_secrets( - self, app: Application, relation: Relation, fields: Optional[List[str]] + self, component: Union[Application, Unit], relation: Relation, fields: Optional[List[str]] ) -> Dict[str, str]: """Fetching databag contents when no secrets are involved. Since the Provider's databag is the only one holding secrest, we can apply a simplified workflow to read the Require's side's databag. - This is used typically when the Provides side wants to read the Requires side's data, + This is used typically when the Provider side wants to read the Requires side's data, or when the Requires side may want to read its own data. """ - if app not in relation.data or not relation.data[app]: + if component not in relation.data or not relation.data[component]: return {} if fields: - return {k: relation.data[app][k] for k in fields if k in relation.data[app]} + return { + k: relation.data[component][k] for k in fields if k in relation.data[component] + } else: - return dict(relation.data[app]) + return dict(relation.data[component]) def _fetch_relation_data_with_secrets( self, - app: Application, + component: Union[Application, Unit], req_secret_fields: Optional[List[str]], relation: Relation, fields: Optional[List[str]] = None, @@ -835,23 +1101,19 @@ def _fetch_relation_data_with_secrets( This function has internal logic to resolve if a requested field may be "hidden" within a Relation Secret, or directly available as a databag field. Typically - used to read the Provides side's databag (eigher by the Requires side, or by - Provides side itself). + used to read the Provider side's databag (eigher by the Requires side, or by + Provider side itself). """ result = {} normal_fields = [] if not fields: - if app not in relation.data or not relation.data[app]: + if component not in relation.data: return {} - all_fields = list(relation.data[app].keys()) + all_fields = list(relation.data[component].keys()) normal_fields = [field for field in all_fields if not self._is_secret_field(field)] - - # There must have been secrets there - if all_fields != normal_fields and req_secret_fields: - # So we assemble the full fields list (without 'secret-' fields) - fields = normal_fields + req_secret_fields + fields = normal_fields + req_secret_fields if req_secret_fields else normal_fields if fields: result, normal_fields = self._process_secret_fields( @@ -859,50 +1121,51 @@ def _fetch_relation_data_with_secrets( ) # Processing "normal" fields. May include leftover from what we couldn't retrieve as a secret. - # (Typically when Juju3 Requires meets Juju2 Provides) + # (Typically when Juju3 Requires meets Juju2 Provider) if normal_fields: result.update( - self._fetch_relation_data_without_secrets(app, relation, list(normal_fields)) + self._fetch_relation_data_without_secrets(component, relation, list(normal_fields)) ) return result def _update_relation_data_without_secrets( - self, app: Application, relation: Relation, data: Dict[str, str] + self, component: Union[Application, Unit], relation: Relation, data: Dict[str, str] ) -> None: """Updating databag contents when no secrets are involved.""" - if app not in relation.data or relation.data[app] is None: + if component not in relation.data or relation.data[component] is None: return - if any(self._is_secret_field(key) for key in data.keys()): - raise SecretsIllegalUpdateError("Can't update secret {key}.") - if relation: - relation.data[app].update(data) + relation.data[component].update(data) def _delete_relation_data_without_secrets( - self, app: Application, relation: Relation, fields: List[str] + self, component: Union[Application, Unit], relation: Relation, fields: List[str] ) -> None: """Remove databag fields 'fields' from Relation.""" - if app not in relation.data or not relation.data[app]: + if component not in relation.data or relation.data[component] is None: return for field in fields: try: - relation.data[app].pop(field) + relation.data[component].pop(field) except KeyError: logger.debug( - "Non-existing field was attempted to be removed from the databag %s, %s", - str(relation.id), + "Non-existing field '%s' was attempted to be removed from the databag (relation ID: %s)", str(field), + str(relation.id), ) pass # Public interface methods # Handling Relation Fields seamlessly, regardless if in databag or a Juju Secret + def as_dict(self, relation_id: int) -> UserDict: + """Dict behavior representation of the Abstract Data.""" + return DataDict(self, relation_id) + def get_relation(self, relation_name, relation_id) -> Relation: """Safe way of retrieving a relation.""" - relation = self.charm.model.get_relation(relation_name, relation_id) + relation = self._model.get_relation(relation_name, relation_id) if not relation: raise DataInterfacesError( @@ -954,7 +1217,6 @@ def fetch_relation_field( .get(field) ) - @leader_only def fetch_my_relation_data( self, relation_ids: Optional[List[int]] = None, @@ -983,7 +1245,6 @@ def fetch_my_relation_data( data[relation.id] = self._fetch_my_specific_relation_data(relation, fields) return data - @leader_only def fetch_my_relation_field( self, relation_id: int, field: str, relation_name: Optional[str] = None ) -> Optional[str]: @@ -1010,14 +1271,22 @@ def delete_relation_data(self, relation_id: int, fields: List[str]) -> None: return self._delete_relation_data(relation, fields) -# Base DataProvides and DataRequires +class EventHandlers(Object): + """Requires-side of the relation.""" + def __init__(self, charm: CharmBase, relation_data: Data, unique_key: str = ""): + """Manager of base client relations.""" + if not unique_key: + unique_key = relation_data.relation_name + super().__init__(charm, unique_key) -class DataProvides(DataRelation): - """Base provides-side of the data products relation.""" + self.charm = charm + self.relation_data = relation_data - def __init__(self, charm: CharmBase, relation_name: str) -> None: - super().__init__(charm, relation_name) + self.framework.observe( + charm.on[self.relation_data.relation_name].relation_changed, + self._on_relation_changed_event, + ) def _diff(self, event: RelationChangedEvent) -> Diff: """Retrieves the diff of the data in the relation changed databag. @@ -1029,33 +1298,64 @@ def _diff(self, event: RelationChangedEvent) -> Diff: a Diff instance containing the added, deleted and changed keys from the event relation databag. """ - return diff(event, self.local_app) + return diff(event, self.relation_data.data_component) + + @abstractmethod + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation data has changed.""" + raise NotImplementedError + + +# Base ProviderData and RequiresData + + +class ProviderData(Data): + """Base provides-side of the data products relation.""" + + def __init__( + self, + model: Model, + relation_name: str, + ) -> None: + super().__init__(model, relation_name) + self.data_component = self.local_app # Private methods handling secrets @juju_secrets_only def _add_relation_secret( - self, relation: Relation, content: Dict[str, str], group_mapping: SecretGroup + self, + relation: Relation, + group_mapping: SecretGroup, + secret_fields: Set[str], + data: Dict[str, str], + uri_to_databag=True, ) -> bool: """Add a new Juju Secret that will be registered in the relation databag.""" secret_field = self._generate_secret_field_name(group_mapping) - if relation.data[self.local_app].get(secret_field): + if uri_to_databag and relation.data[self.component].get(secret_field): logging.error("Secret for relation %s already exists, not adding again", relation.id) return False + content = self._content_for_secret_group(data, secret_fields, group_mapping) + label = self._generate_secret_label(self.relation_name, relation.id, group_mapping) secret = self.secrets.add(label, content, relation) # According to lint we may not have a Secret ID - if secret.meta and secret.meta.id: - relation.data[self.local_app][secret_field] = secret.meta.id + if uri_to_databag and secret.meta and secret.meta.id: + relation.data[self.component][secret_field] = secret.meta.id # Return the content that was added return True @juju_secrets_only def _update_relation_secret( - self, relation: Relation, content: Dict[str, str], group_mapping: SecretGroup + self, + relation: Relation, + group_mapping: SecretGroup, + secret_fields: Set[str], + data: Dict[str, str], ) -> bool: """Update the contents of an existing Juju Secret, referred in the relation databag.""" secret = self._get_relation_secret(relation.id, group_mapping) @@ -1064,6 +1364,8 @@ def _update_relation_secret( logging.error("Can't update secret for relation %s", relation.id) return False + content = self._content_for_secret_group(data, secret_fields, group_mapping) + old_content = secret.get_content() full_content = copy.deepcopy(old_content) full_content.update(content) @@ -1078,13 +1380,13 @@ def _add_or_update_relation_secrets( group: SecretGroup, secret_fields: Set[str], data: Dict[str, str], + uri_to_databag=True, ) -> bool: """Update contents for Secret group. If the Secret doesn't exist, create it.""" - secret_content = self._content_for_secret_group(data, secret_fields, group) if self._get_relation_secret(relation.id, group): - return self._update_relation_secret(relation, secret_content, group) + return self._update_relation_secret(relation, group, secret_fields, data) else: - return self._add_relation_secret(relation, secret_content, group) + return self._add_relation_secret(relation, group, secret_fields, data, uri_to_databag) @juju_secrets_only def _delete_relation_secret( @@ -1103,22 +1405,24 @@ def _delete_relation_secret( try: new_content.pop(field) except KeyError: - logging.error( + logging.debug( "Non-existing secret was attempted to be removed %s, %s", str(relation.id), str(field), ) return False - secret.set_content(new_content) - # Remove secret from the relation if it's fully gone if not new_content: field = self._generate_secret_field_name(group) try: - relation.data[self.local_app].pop(field) + relation.data[self.component].pop(field) except KeyError: pass + label = self._generate_secret_label(self.relation_name, relation.id, group) + self.secrets.remove(label) + else: + secret.set_content(new_content) # Return the content that was removed return True @@ -1137,7 +1441,7 @@ def _get_relation_secret( if secret := self.secrets.get(label): return secret - relation = self.charm.model.get_relation(relation_name, relation_id) + relation = self._model.get_relation(relation_name, relation_id) if not relation: return @@ -1148,9 +1452,9 @@ def _get_relation_secret( def _fetch_specific_relation_data( self, relation: Relation, fields: Optional[List[str]] ) -> Dict[str, str]: - """Fetching relation data for Provides. + """Fetching relation data for Provider. - NOTE: Since all secret fields are in the Provides side of the databag, we don't need to worry about that + NOTE: Since all secret fields are in the Provider side of the databag, we don't need to worry about that """ if not relation.app: return {} @@ -1233,33 +1537,31 @@ def set_tls_ca(self, relation_id: int, tls_ca: str) -> None: """ self.update_relation_data(relation_id, {"tls-ca": tls_ca}) + # Public functions -- inherited -class DataRequires(DataRelation): - """Requires-side of the relation.""" + fetch_my_relation_data = leader_only(Data.fetch_my_relation_data) + fetch_my_relation_field = leader_only(Data.fetch_my_relation_field) + + +class RequirerData(Data): + """Requirer-side of the relation.""" SECRET_FIELDS = ["username", "password", "tls", "tls-ca", "uris"] def __init__( self, - charm, + model, relation_name: str, extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], ): """Manager of base client relations.""" - super().__init__(charm, relation_name) + super().__init__(model, relation_name) self.extra_user_roles = extra_user_roles self._secret_fields = list(self.SECRET_FIELDS) if additional_secret_fields: self._secret_fields += additional_secret_fields - - self.framework.observe( - self.charm.on[relation_name].relation_created, self._on_relation_created_event - ) - self.framework.observe( - charm.on.secret_changed, - self._on_secret_changed_event, - ) + self.data_component = self.local_unit @property def secret_fields(self) -> Optional[List[str]]: @@ -1267,18 +1569,6 @@ def secret_fields(self) -> Optional[List[str]]: if self.secrets_enabled: return self._secret_fields - def _diff(self, event: RelationChangedEvent) -> Diff: - """Retrieves the diff of the data in the relation changed databag. - - Args: - event: relation changed event. - - Returns: - a Diff instance containing the added, deleted and changed - keys from the event relation databag. - """ - return diff(event, self.local_unit) - # Internal helper functions def _register_secret_to_relation( @@ -1291,13 +1581,13 @@ def _register_secret_to_relation( then will be "stuck" on the Secret object, whenever it may appear (i.e. as an event attribute, or fetched manually) on future occasions. - This will allow us to uniquely identify the secret on Provides side (typically on + This will allow us to uniquely identify the secret on Provider side (typically on 'secret-changed' events), and map it to the corresponding relation. """ label = self._generate_secret_label(relation_name, relation_id, group) # Fetchin the Secret's meta information ensuring that it's locally getting registered with - CachedSecret(self.charm, label, secret_id).meta + CachedSecret(self._model, self.component, label, secret_id).meta def _register_secrets_to_relation(self, relation: Relation, params_name_list: List[str]): """Make sure that secrets of the provided list are locally 'registered' from the databag. @@ -1307,7 +1597,7 @@ def _register_secrets_to_relation(self, relation: Relation, params_name_list: Li if not relation.app: return - for group in SecretGroup: + for group in SECRET_GROUPS.groups(): secret_field = self._generate_secret_field_name(group) if secret_field in params_name_list: if secret_uri := relation.data[relation.app].get(secret_field): @@ -1357,23 +1647,6 @@ def is_resource_created(self, relation_id: Optional[int] = None) -> bool: else False ) - # Event handlers - - def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: - """Event emitted when the relation is created.""" - if not self.local_unit.is_leader(): - return - - if self.secret_fields: - set_encoded_field( - event.relation, self.charm.app, REQ_SECRET_FIELDS, self.secret_fields - ) - - @abstractmethod - def _on_secret_changed_event(self, event: RelationChangedEvent) -> None: - """Event emitted when the relation data has changed.""" - raise NotImplementedError - # Mandatory internal overrides @juju_secrets_only @@ -1390,7 +1663,7 @@ def _get_relation_secret( def _fetch_specific_relation_data( self, relation, fields: Optional[List[str]] = None ) -> Dict[str, str]: - """Fetching Requires data -- that may include secrets.""" + """Fetching Requirer data -- that may include secrets.""" if not relation.app: return {} return self._fetch_relation_data_with_secrets( @@ -1426,73 +1699,645 @@ def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: """ return self._delete_relation_data_without_secrets(self.local_app, relation, fields) + # Public functions -- inherited -# General events + fetch_my_relation_data = leader_only(Data.fetch_my_relation_data) + fetch_my_relation_field = leader_only(Data.fetch_my_relation_field) -class ExtraRoleEvent(RelationEvent): - """Base class for data events.""" +class RequirerEventHandlers(EventHandlers): + """Requires-side of the relation.""" - @property - def extra_user_roles(self) -> Optional[str]: - """Returns the extra user roles that were requested.""" - if not self.relation.app: - return None + def __init__(self, charm: CharmBase, relation_data: RequirerData, unique_key: str = ""): + """Manager of base client relations.""" + super().__init__(charm, relation_data, unique_key) - return self.relation.data[self.relation.app].get("extra-user-roles") + self.framework.observe( + self.charm.on[relation_data.relation_name].relation_created, + self._on_relation_created_event, + ) + self.framework.observe( + charm.on.secret_changed, + self._on_secret_changed_event, + ) + # Event handlers -class AuthenticationEvent(RelationEvent): - """Base class for authentication fields for events. + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the relation is created.""" + if not self.relation_data.local_unit.is_leader(): + return - The amount of logic added here is not ideal -- but this was the only way to preserve - the interface when moving to Juju Secrets - """ + if self.relation_data.secret_fields: # pyright: ignore [reportAttributeAccessIssue] + set_encoded_field( + event.relation, + self.relation_data.component, + REQ_SECRET_FIELDS, + self.relation_data.secret_fields, # pyright: ignore [reportAttributeAccessIssue] + ) - @property - def _secrets(self) -> dict: - """Caching secrets to avoid fetching them each time a field is referrd. + @abstractmethod + def _on_secret_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation data has changed.""" + raise NotImplementedError - DON'T USE the encapsulated helper variable outside of this function - """ - if not hasattr(self, "_cached_secrets"): - self._cached_secrets = {} - return self._cached_secrets - @property - def _jujuversion(self) -> JujuVersion: - """Caching jujuversion to avoid a Juju call on each field evaluation. +################################################################################ +# Peer Relation Data +################################################################################ - DON'T USE the encapsulated helper variable outside of this function - """ - if not hasattr(self, "_cached_jujuversion"): - self._cached_jujuversion = None - if not self._cached_jujuversion: - self._cached_jujuversion = JujuVersion.from_environ() - return self._cached_jujuversion - def _get_secret(self, group) -> Optional[Dict[str, str]]: - """Retrieveing secrets.""" - if not self.app: - return - if not self._secrets.get(group): - self._secrets[group] = None - secret_field = f"{PROV_SECRET_PREFIX}{group}" - if secret_uri := self.relation.data[self.app].get(secret_field): - secret = self.framework.model.get_secret(id=secret_uri) - self._secrets[group] = secret.get_content() - return self._secrets[group] +class DataPeerData(RequirerData, ProviderData): + """Represents peer relations data.""" - @property - def secrets_enabled(self): - """Is this Juju version allowing for Secrets usage?""" - return self._jujuversion.has_secrets + SECRET_FIELDS = [] + SECRET_FIELD_NAME = "internal_secret" + SECRET_LABEL_MAP = {} - @property - def username(self) -> Optional[str]: - """Returns the created username.""" - if not self.relation.app: - return None + def __init__( + self, + model, + relation_name: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + additional_secret_group_mapping: Dict[str, str] = {}, + secret_field_name: Optional[str] = None, + deleted_label: Optional[str] = None, + ): + """Manager of base client relations.""" + RequirerData.__init__( + self, + model, + relation_name, + extra_user_roles, + additional_secret_fields, + ) + self.secret_field_name = secret_field_name if secret_field_name else self.SECRET_FIELD_NAME + self.deleted_label = deleted_label + self._secret_label_map = {} + # Secrets that are being dynamically added within the scope of this event handler run + self._new_secrets = [] + self._additional_secret_group_mapping = additional_secret_group_mapping + + for group, fields in additional_secret_group_mapping.items(): + if group not in SECRET_GROUPS.groups(): + setattr(SECRET_GROUPS, group, group) + for field in fields: + secret_group = SECRET_GROUPS.get_group(group) + internal_field = self._field_to_internal_name(field, secret_group) + self._secret_label_map.setdefault(group, []).append(internal_field) + self._secret_fields.append(internal_field) + + @property + def scope(self) -> Optional[Scope]: + """Turn component information into Scope.""" + if isinstance(self.component, Application): + return Scope.APP + if isinstance(self.component, Unit): + return Scope.UNIT + + @property + def secret_label_map(self) -> Dict[str, str]: + """Property storing secret mappings.""" + return self._secret_label_map + + @property + def static_secret_fields(self) -> List[str]: + """Re-definition of the property in a way that dynamically extended list is retrieved.""" + return self._secret_fields + + @property + def secret_fields(self) -> List[str]: + """Re-definition of the property in a way that dynamically extended list is retrieved.""" + return ( + self.static_secret_fields if self.static_secret_fields else self.current_secret_fields + ) + + @property + def current_secret_fields(self) -> List[str]: + """Helper method to get all currently existing secret fields (added statically or dynamically).""" + if not self.secrets_enabled: + return [] + + if len(self._model.relations[self.relation_name]) > 1: + raise ValueError(f"More than one peer relation on {self.relation_name}") + + relation = self._model.relations[self.relation_name][0] + fields = [] + + ignores = [SECRET_GROUPS.get_group("user"), SECRET_GROUPS.get_group("tls")] + for group in SECRET_GROUPS.groups(): + if group in ignores: + continue + if content := self._get_group_secret_contents(relation, group): + fields += list(content.keys()) + return list(set(fields) | set(self._new_secrets)) + + @dynamic_secrets_only + def set_secret( + self, + relation_id: int, + field: str, + value: str, + group_mapping: Optional[SecretGroup] = None, + ) -> None: + """Public interface method to add a Relation Data field specifically as a Juju Secret. + + Args: + relation_id: ID of the relation + field: The secret field that is to be added + value: The string value of the secret + group_mapping: The name of the "secret group", in case the field is to be added to an existing secret + """ + full_field = self._field_to_internal_name(field, group_mapping) + if self.secrets_enabled and full_field not in self.current_secret_fields: + self._new_secrets.append(full_field) + if self._no_group_with_databag(field, full_field): + self.update_relation_data(relation_id, {full_field: value}) + + # Unlike for set_secret(), there's no harm using this operation with static secrets + # The restricion is only added to keep the concept clear + @dynamic_secrets_only + def get_secret( + self, + relation_id: int, + field: str, + group_mapping: Optional[SecretGroup] = None, + ) -> Optional[str]: + """Public interface method to fetch secrets only.""" + full_field = self._field_to_internal_name(field, group_mapping) + if ( + self.secrets_enabled + and full_field not in self.current_secret_fields + and field not in self.current_secret_fields + ): + return + if self._no_group_with_databag(field, full_field): + return self.fetch_my_relation_field(relation_id, full_field) + + @dynamic_secrets_only + def delete_secret( + self, + relation_id: int, + field: str, + group_mapping: Optional[SecretGroup] = None, + ) -> Optional[str]: + """Public interface method to delete secrets only.""" + full_field = self._field_to_internal_name(field, group_mapping) + if self.secrets_enabled and full_field not in self.current_secret_fields: + logger.warning(f"Secret {field} from group {group_mapping} was not found") + return + if self._no_group_with_databag(field, full_field): + self.delete_relation_data(relation_id, [full_field]) + + # Helpers + + @staticmethod + def _field_to_internal_name(field: str, group: Optional[SecretGroup]) -> str: + if not group or group == SECRET_GROUPS.EXTRA: + return field + return f"{field}{GROUP_SEPARATOR}{group}" + + @staticmethod + def _internal_name_to_field(name: str) -> Tuple[str, SecretGroup]: + parts = name.split(GROUP_SEPARATOR) + if not len(parts) > 1: + return (parts[0], SECRET_GROUPS.EXTRA) + secret_group = SECRET_GROUPS.get_group(parts[1]) + if not secret_group: + raise ValueError(f"Invalid secret field {name}") + return (parts[0], secret_group) + + def _group_secret_fields(self, secret_fields: List[str]) -> Dict[SecretGroup, List[str]]: + """Helper function to arrange secret mappings under their group. + + NOTE: All unrecognized items end up in the 'extra' secret bucket. + Make sure only secret fields are passed! + """ + secret_fieldnames_grouped = {} + for key in secret_fields: + field, group = self._internal_name_to_field(key) + secret_fieldnames_grouped.setdefault(group, []).append(field) + return secret_fieldnames_grouped + + def _content_for_secret_group( + self, content: Dict[str, str], secret_fields: Set[str], group_mapping: SecretGroup + ) -> Dict[str, str]: + """Select : pairs from input, that belong to this particular Secret group.""" + if group_mapping == SECRET_GROUPS.EXTRA: + return {k: v for k, v in content.items() if k in self.secret_fields} + return { + self._internal_name_to_field(k)[0]: v + for k, v in content.items() + if k in self.secret_fields + } + + # Backwards compatibility + + def _check_deleted_label(self, relation, fields) -> None: + """Helper function for legacy behavior.""" + current_data = self.fetch_my_relation_data([relation.id], fields) + if current_data is not None: + # Check if the secret we wanna delete actually exists + # Given the "deleted label", here we can't rely on the default mechanism (i.e. 'key not found') + if non_existent := (set(fields) & set(self.secret_fields)) - set( + current_data.get(relation.id, []) + ): + logger.debug( + "Non-existing secret %s was attempted to be removed.", + ", ".join(non_existent), + ) + + def _remove_secret_from_databag(self, relation, fields: List[str]) -> None: + """For Rolling Upgrades -- when moving from databag to secrets usage. + + Practically what happens here is to remove stuff from the databag that is + to be stored in secrets. + """ + if not self.secret_fields: + return + + secret_fields_passed = set(self.secret_fields) & set(fields) + for field in secret_fields_passed: + if self._fetch_relation_data_without_secrets(self.component, relation, [field]): + self._delete_relation_data_without_secrets(self.component, relation, [field]) + + def _remove_secret_field_name_from_databag(self, relation) -> None: + """Making sure that the old databag URI is gone. + + This action should not be executed more than once. + """ + # Nothing to do if 'internal-secret' is not in the databag + if not (relation.data[self.component].get(self._generate_secret_field_name())): + return + + # Making sure that the secret receives its label + # (This should have happened by the time we get here, rather an extra security measure.) + secret = self._get_relation_secret(relation.id) + + # Either app scope secret with leader executing, or unit scope secret + leader_or_unit_scope = self.component != self.local_app or self.local_unit.is_leader() + if secret and leader_or_unit_scope: + # Databag reference to the secret URI can be removed, now that it's labelled + relation.data[self.component].pop(self._generate_secret_field_name(), None) + + def _previous_labels(self) -> List[str]: + """Generator for legacy secret label names, for backwards compatibility.""" + result = [] + members = [self._model.app.name] + if self.scope: + members.append(self.scope.value) + result.append(f"{'.'.join(members)}") + return result + + def _no_group_with_databag(self, field: str, full_field: str) -> bool: + """Check that no secret group is attempted to be used together with databag.""" + if not self.secrets_enabled and full_field != field: + logger.error( + f"Can't access {full_field}: no secrets available (i.e. no secret groups either)." + ) + return False + return True + + # Event handlers + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + pass + + def _on_secret_changed_event(self, event: SecretChangedEvent) -> None: + """Event emitted when the secret has changed.""" + pass + + # Overrides of Relation Data handling functions + + def _generate_secret_label( + self, relation_name: str, relation_id: int, group_mapping: SecretGroup + ) -> str: + members = [relation_name, self._model.app.name] + if self.scope: + members.append(self.scope.value) + if group_mapping != SECRET_GROUPS.EXTRA: + members.append(group_mapping) + return f"{'.'.join(members)}" + + def _generate_secret_field_name(self, group_mapping: SecretGroup = SECRET_GROUPS.EXTRA) -> str: + """Generate unique group_mappings for secrets within a relation context.""" + return f"{self.secret_field_name}" + + @juju_secrets_only + def _get_relation_secret( + self, + relation_id: int, + group_mapping: SecretGroup = SECRET_GROUPS.EXTRA, + relation_name: Optional[str] = None, + ) -> Optional[CachedSecret]: + """Retrieve a Juju Secret specifically for peer relations. + + In case this code may be executed within a rolling upgrade, and we may need to + migrate secrets from the databag to labels, we make sure to stick the correct + label on the secret, and clean up the local databag. + """ + if not relation_name: + relation_name = self.relation_name + + relation = self._model.get_relation(relation_name, relation_id) + if not relation: + return + + label = self._generate_secret_label(relation_name, relation_id, group_mapping) + secret_uri = relation.data[self.component].get(self._generate_secret_field_name(), None) + + # URI or legacy label is only to applied when moving single legacy secret to a (new) label + if group_mapping == SECRET_GROUPS.EXTRA: + # Fetching the secret with fallback to URI (in case label is not yet known) + # Label would we "stuck" on the secret in case it is found + return self.secrets.get(label, secret_uri, legacy_labels=self._previous_labels()) + return self.secrets.get(label) + + def _get_group_secret_contents( + self, + relation: Relation, + group: SecretGroup, + secret_fields: Union[Set[str], List[str]] = [], + ) -> Dict[str, str]: + """Helper function to retrieve collective, requested contents of a secret.""" + secret_fields = [self._internal_name_to_field(k)[0] for k in secret_fields] + result = super()._get_group_secret_contents(relation, group, secret_fields) + if self.deleted_label: + result = {key: result[key] for key in result if result[key] != self.deleted_label} + if self._additional_secret_group_mapping: + return {self._field_to_internal_name(key, group): result[key] for key in result} + return result + + @either_static_or_dynamic_secrets + def _fetch_my_specific_relation_data( + self, relation: Relation, fields: Optional[List[str]] + ) -> Dict[str, str]: + """Fetch data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + return self._fetch_relation_data_with_secrets( + self.component, self.secret_fields, relation, fields + ) + + @either_static_or_dynamic_secrets + def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: + """Update data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + self._remove_secret_from_databag(relation, list(data.keys())) + _, normal_fields = self._process_secret_fields( + relation, + self.secret_fields, + list(data), + self._add_or_update_relation_secrets, + data=data, + uri_to_databag=False, + ) + self._remove_secret_field_name_from_databag(relation) + + normal_content = {k: v for k, v in data.items() if k in normal_fields} + self._update_relation_data_without_secrets(self.component, relation, normal_content) + + @either_static_or_dynamic_secrets + def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: + """Delete data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + if self.secret_fields and self.deleted_label: + # Legacy, backwards compatibility + self._check_deleted_label(relation, fields) + + _, normal_fields = self._process_secret_fields( + relation, + self.secret_fields, + fields, + self._update_relation_secret, + data={field: self.deleted_label for field in fields}, + ) + else: + _, normal_fields = self._process_secret_fields( + relation, self.secret_fields, fields, self._delete_relation_secret, fields=fields + ) + self._delete_relation_data_without_secrets(self.component, relation, list(normal_fields)) + + def fetch_relation_data( + self, + relation_ids: Optional[List[int]] = None, + fields: Optional[List[str]] = None, + relation_name: Optional[str] = None, + ) -> Dict[int, Dict[str, str]]: + """This method makes no sense for a Peer Relation.""" + raise NotImplementedError( + "Peer Relation only supports 'self-side' fetch methods: " + "fetch_my_relation_data() and fetch_my_relation_field()" + ) + + def fetch_relation_field( + self, relation_id: int, field: str, relation_name: Optional[str] = None + ) -> Optional[str]: + """This method makes no sense for a Peer Relation.""" + raise NotImplementedError( + "Peer Relation only supports 'self-side' fetch methods: " + "fetch_my_relation_data() and fetch_my_relation_field()" + ) + + # Public functions -- inherited + + fetch_my_relation_data = Data.fetch_my_relation_data + fetch_my_relation_field = Data.fetch_my_relation_field + + +class DataPeerEventHandlers(RequirerEventHandlers): + """Requires-side of the relation.""" + + def __init__(self, charm: CharmBase, relation_data: RequirerData, unique_key: str = ""): + """Manager of base client relations.""" + super().__init__(charm, relation_data, unique_key) + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + pass + + def _on_secret_changed_event(self, event: SecretChangedEvent) -> None: + """Event emitted when the secret has changed.""" + pass + + +class DataPeer(DataPeerData, DataPeerEventHandlers): + """Represents peer relations.""" + + def __init__( + self, + charm, + relation_name: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + additional_secret_group_mapping: Dict[str, str] = {}, + secret_field_name: Optional[str] = None, + deleted_label: Optional[str] = None, + unique_key: str = "", + ): + DataPeerData.__init__( + self, + charm.model, + relation_name, + extra_user_roles, + additional_secret_fields, + additional_secret_group_mapping, + secret_field_name, + deleted_label, + ) + DataPeerEventHandlers.__init__(self, charm, self, unique_key) + + +class DataPeerUnitData(DataPeerData): + """Unit data abstraction representation.""" + + SCOPE = Scope.UNIT + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class DataPeerUnit(DataPeerUnitData, DataPeerEventHandlers): + """Unit databag representation.""" + + def __init__( + self, + charm, + relation_name: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + additional_secret_group_mapping: Dict[str, str] = {}, + secret_field_name: Optional[str] = None, + deleted_label: Optional[str] = None, + unique_key: str = "", + ): + DataPeerData.__init__( + self, + charm.model, + relation_name, + extra_user_roles, + additional_secret_fields, + additional_secret_group_mapping, + secret_field_name, + deleted_label, + ) + DataPeerEventHandlers.__init__(self, charm, self, unique_key) + + +class DataPeerOtherUnitData(DataPeerUnitData): + """Unit data abstraction representation.""" + + def __init__(self, unit: Unit, *args, **kwargs): + super().__init__(*args, **kwargs) + self.local_unit = unit + self.component = unit + + def update_relation_data(self, relation_id: int, data: dict) -> None: + """This method makes no sense for a Other Peer Relation.""" + raise NotImplementedError("It's not possible to update data of another unit.") + + def delete_relation_data(self, relation_id: int, fields: List[str]) -> None: + """This method makes no sense for a Other Peer Relation.""" + raise NotImplementedError("It's not possible to delete data of another unit.") + + +class DataPeerOtherUnitEventHandlers(DataPeerEventHandlers): + """Requires-side of the relation.""" + + def __init__(self, charm: CharmBase, relation_data: DataPeerUnitData): + """Manager of base client relations.""" + unique_key = f"{relation_data.relation_name}-{relation_data.local_unit.name}" + super().__init__(charm, relation_data, unique_key=unique_key) + + +class DataPeerOtherUnit(DataPeerOtherUnitData, DataPeerOtherUnitEventHandlers): + """Unit databag representation for another unit than the executor.""" + + def __init__( + self, + unit: Unit, + charm: CharmBase, + relation_name: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + additional_secret_group_mapping: Dict[str, str] = {}, + secret_field_name: Optional[str] = None, + deleted_label: Optional[str] = None, + ): + DataPeerOtherUnitData.__init__( + self, + unit, + charm.model, + relation_name, + extra_user_roles, + additional_secret_fields, + additional_secret_group_mapping, + secret_field_name, + deleted_label, + ) + DataPeerOtherUnitEventHandlers.__init__(self, charm, self) + + +################################################################################ +# Cross-charm Relatoins Data Handling and Evenets +################################################################################ + +# Generic events + + +class ExtraRoleEvent(RelationEvent): + """Base class for data events.""" + + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-user-roles") + + +class RelationEventWithSecret(RelationEvent): + """Base class for Relation Events that need to handle secrets.""" + + @property + def _secrets(self) -> dict: + """Caching secrets to avoid fetching them each time a field is referrd. + + DON'T USE the encapsulated helper variable outside of this function + """ + if not hasattr(self, "_cached_secrets"): + self._cached_secrets = {} + return self._cached_secrets + + def _get_secret(self, group) -> Optional[Dict[str, str]]: + """Retrieveing secrets.""" + if not self.app: + return + if not self._secrets.get(group): + self._secrets[group] = None + secret_field = f"{PROV_SECRET_PREFIX}{group}" + if secret_uri := self.relation.data[self.app].get(secret_field): + secret = self.framework.model.get_secret(id=secret_uri) + self._secrets[group] = secret.get_content() + return self._secrets[group] + + @property + def secrets_enabled(self): + """Is this Juju version allowing for Secrets usage?""" + return JujuVersion.from_environ().has_secrets + + +class AuthenticationEvent(RelationEventWithSecret): + """Base class for authentication fields for events. + + The amount of logic added here is not ideal -- but this was the only way to preserve + the interface when moving to Juju Secrets + """ + + @property + def username(self) -> Optional[str]: + """Returns the created username.""" + if not self.relation.app: + return None if self.secrets_enabled: secret = self._get_secret("user") @@ -1559,6 +2404,17 @@ def database(self) -> Optional[str]: class DatabaseRequestedEvent(DatabaseProvidesEvent, ExtraRoleEvent): """Event emitted when a new database is requested for use on this relation.""" + @property + def external_node_connectivity(self) -> bool: + """Returns the requested external_node_connectivity field.""" + if not self.relation.app: + return False + + return ( + self.relation.data[self.relation.app].get("external-node-connectivity", "false") + == "true" + ) + class DatabaseProvidesEvents(CharmEvents): """Database events. @@ -1569,7 +2425,7 @@ class DatabaseProvidesEvents(CharmEvents): database_requested = EventSource(DatabaseRequestedEvent) -class DatabaseRequiresEvent(RelationEvent): +class DatabaseRequiresEvent(RelationEventWithSecret): """Base class for database events.""" @property @@ -1624,6 +2480,11 @@ def uris(self) -> Optional[str]: if not self.relation.app: return None + if self.secrets_enabled: + secret = self._get_secret("user") + if secret: + return secret.get("uris") + return self.relation.data[self.relation.app].get("uris") @property @@ -1664,28 +2525,11 @@ class DatabaseRequiresEvents(CharmEvents): # Database Provider and Requires -class DatabaseProvides(DataProvides): - """Provider-side of the database relations.""" - - on = DatabaseProvidesEvents() # pyright: ignore [reportGeneralTypeIssues] - - def __init__(self, charm: CharmBase, relation_name: str) -> None: - super().__init__(charm, relation_name) - - def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: - """Event emitted when the relation has changed.""" - # Leader only - if not self.local_unit.is_leader(): - return - # Check which data has changed to emit customs events. - diff = self._diff(event) +class DatabaseProviderData(ProviderData): + """Provider-side data of the database relations.""" - # Emit a database requested event if the setup key (database name and optional - # extra user roles) was added to the relation databag by the application. - if "database" in diff.added: - getattr(self.on, "database_requested").emit( - event.relation, app=event.app, unit=event.unit - ) + def __init__(self, model: Model, relation_name: str) -> None: + super().__init__(model, relation_name) def set_database(self, relation_id: int, database_name: str) -> None: """Set database name. @@ -1759,37 +2603,140 @@ def set_version(self, relation_id: int, version: str) -> None: self.update_relation_data(relation_id, {"version": version}) -class DatabaseRequires(DataRequires): - """Requires-side of the database relation.""" +class DatabaseProviderEventHandlers(EventHandlers): + """Provider-side of the database relation handlers.""" + + on = DatabaseProvidesEvents() # pyright: ignore [reportAssignmentType] + + def __init__( + self, charm: CharmBase, relation_data: DatabaseProviderData, unique_key: str = "" + ): + """Manager of base client relations.""" + super().__init__(charm, relation_data, unique_key) + # Just to calm down pyright, it can't parse that the same type is being used in the super() call above + self.relation_data = relation_data + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Emit a database requested event if the setup key (database name and optional + # extra user roles) was added to the relation databag by the application. + if "database" in diff.added: + getattr(self.on, "database_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + +class DatabaseProvides(DatabaseProviderData, DatabaseProviderEventHandlers): + """Provider-side of the database relations.""" + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + DatabaseProviderData.__init__(self, charm.model, relation_name) + DatabaseProviderEventHandlers.__init__(self, charm, self) + - on = DatabaseRequiresEvents() # pyright: ignore [reportGeneralTypeIssues] +class DatabaseRequirerData(RequirerData): + """Requirer-side of the database relation.""" def __init__( self, - charm, + model: Model, relation_name: str, database_name: str, extra_user_roles: Optional[str] = None, relations_aliases: Optional[List[str]] = None, additional_secret_fields: Optional[List[str]] = [], + external_node_connectivity: bool = False, ): """Manager of database client relations.""" - super().__init__(charm, relation_name, extra_user_roles, additional_secret_fields) + super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) self.database = database_name self.relations_aliases = relations_aliases + self.external_node_connectivity = external_node_connectivity + + def is_postgresql_plugin_enabled(self, plugin: str, relation_index: int = 0) -> bool: + """Returns whether a plugin is enabled in the database. + + Args: + plugin: name of the plugin to check. + relation_index: optional relation index to check the database + (default: 0 - first relation). + + PostgreSQL only. + """ + # Psycopg 3 is imported locally to avoid the need of its package installation + # when relating to a database charm other than PostgreSQL. + import psycopg + + # Return False if no relation is established. + if len(self.relations) == 0: + return False + + relation_id = self.relations[relation_index].id + host = self.fetch_relation_field(relation_id, "endpoints") + + # Return False if there is no endpoint available. + if host is None: + return False + + host = host.split(":")[0] + + content = self.fetch_relation_data([relation_id], ["username", "password"]).get( + relation_id, {} + ) + user = content.get("username") + password = content.get("password") + + connection_string = ( + f"host='{host}' dbname='{self.database}' user='{user}' password='{password}'" + ) + try: + with psycopg.connect(connection_string) as connection: + with connection.cursor() as cursor: + cursor.execute( + "SELECT TRUE FROM pg_extension WHERE extname=%s::text;", (plugin,) + ) + return cursor.fetchone() is not None + except psycopg.Error as e: + logger.exception( + f"failed to check whether {plugin} plugin is enabled in the database: %s", str(e) + ) + return False + + +class DatabaseRequirerEventHandlers(RequirerEventHandlers): + """Requires-side of the relation.""" + + on = DatabaseRequiresEvents() # pyright: ignore [reportAssignmentType] + + def __init__( + self, charm: CharmBase, relation_data: DatabaseRequirerData, unique_key: str = "" + ): + """Manager of base client relations.""" + super().__init__(charm, relation_data, unique_key) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data # Define custom event names for each alias. - if relations_aliases: + if self.relation_data.relations_aliases: # Ensure the number of aliases does not exceed the maximum # of connections allowed in the specific relation. - relation_connection_limit = self.charm.meta.requires[relation_name].limit - if len(relations_aliases) != relation_connection_limit: + relation_connection_limit = self.charm.meta.requires[ + self.relation_data.relation_name + ].limit + if len(self.relation_data.relations_aliases) != relation_connection_limit: raise ValueError( f"The number of aliases must match the maximum number of connections allowed in the relation. " - f"Expected {relation_connection_limit}, got {len(relations_aliases)}" + f"Expected {relation_connection_limit}, got {len(self.relation_data.relations_aliases)}" ) - for relation_alias in relations_aliases: + if self.relation_data.relations_aliases: + for relation_alias in self.relation_data.relations_aliases: self.on.define_event(f"{relation_alias}_database_created", DatabaseCreatedEvent) self.on.define_event( f"{relation_alias}_endpoints_changed", DatabaseEndpointsChangedEvent @@ -1812,32 +2759,32 @@ def _assign_relation_alias(self, relation_id: int) -> None: relation_id: the identifier for a particular relation. """ # If no aliases were provided, return immediately. - if not self.relations_aliases: + if not self.relation_data.relations_aliases: return # Return if an alias was already assigned to this relation # (like when there are more than one unit joining the relation). - relation = self.charm.model.get_relation(self.relation_name, relation_id) - if relation and relation.data[self.local_unit].get("alias"): + relation = self.charm.model.get_relation(self.relation_data.relation_name, relation_id) + if relation and relation.data[self.relation_data.local_unit].get("alias"): return # Retrieve the available aliases (the ones that weren't assigned to any relation). - available_aliases = self.relations_aliases[:] - for relation in self.charm.model.relations[self.relation_name]: - alias = relation.data[self.local_unit].get("alias") + available_aliases = self.relation_data.relations_aliases[:] + for relation in self.charm.model.relations[self.relation_data.relation_name]: + alias = relation.data[self.relation_data.local_unit].get("alias") if alias: logger.debug("Alias %s was already assigned to relation %d", alias, relation.id) available_aliases.remove(alias) # Set the alias in the unit relation databag of the specific relation. - relation = self.charm.model.get_relation(self.relation_name, relation_id) + relation = self.charm.model.get_relation(self.relation_data.relation_name, relation_id) if relation: - relation.data[self.local_unit].update({"alias": available_aliases[0]}) + relation.data[self.relation_data.local_unit].update({"alias": available_aliases[0]}) # We need to set relation alias also on the application level so, # it will be accessible in show-unit juju command, executed for a consumer application unit - if self.local_unit.is_leader(): - self.update_relation_data(relation_id, {"alias": available_aliases[0]}) + if self.relation_data.local_unit.is_leader(): + self.relation_data.update_relation_data(relation_id, {"alias": available_aliases[0]}) def _emit_aliased_event(self, event: RelationChangedEvent, event_name: str) -> None: """Emit an aliased event to a particular relation if it has an alias. @@ -1861,60 +2808,11 @@ def _get_relation_alias(self, relation_id: int) -> Optional[str]: Returns: the relation alias or None if the relation was not found. """ - for relation in self.charm.model.relations[self.relation_name]: + for relation in self.charm.model.relations[self.relation_data.relation_name]: if relation.id == relation_id: - return relation.data[self.local_unit].get("alias") + return relation.data[self.relation_data.local_unit].get("alias") return None - def is_postgresql_plugin_enabled(self, plugin: str, relation_index: int = 0) -> bool: - """Returns whether a plugin is enabled in the database. - - Args: - plugin: name of the plugin to check. - relation_index: optional relation index to check the database - (default: 0 - first relation). - - PostgreSQL only. - """ - # Psycopg 3 is imported locally to avoid the need of its package installation - # when relating to a database charm other than PostgreSQL. - import psycopg - - # Return False if no relation is established. - if len(self.relations) == 0: - return False - - relation_id = self.relations[relation_index].id - host = self.fetch_relation_field(relation_id, "endpoints") - - # Return False if there is no endpoint available. - if host is None: - return False - - host = host.split(":")[0] - - content = self.fetch_relation_data([relation_id], ["username", "password"]).get( - relation_id, {} - ) - user = content.get("username") - password = content.get("password") - - connection_string = ( - f"host='{host}' dbname='{self.database}' user='{user}' password='{password}'" - ) - try: - with psycopg.connect(connection_string) as connection: - with connection.cursor() as cursor: - cursor.execute( - "SELECT TRUE FROM pg_extension WHERE extname=%s::text;", (plugin,) - ) - return cursor.fetchone() is not None - except psycopg.Error as e: - logger.exception( - f"failed to check whether {plugin} plugin is enabled in the database: %s", str(e) - ) - return False - def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: """Event emitted when the database relation is created.""" super()._on_relation_created_event(event) @@ -1924,19 +2822,19 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: # Sets both database and extra user roles in the relation # if the roles are provided. Otherwise, sets only the database. - if not self.local_unit.is_leader(): + if not self.relation_data.local_unit.is_leader(): return - if self.extra_user_roles: - self.update_relation_data( - event.relation.id, - { - "database": self.database, - "extra-user-roles": self.extra_user_roles, - }, - ) - else: - self.update_relation_data(event.relation.id, {"database": self.database}) + event_data = {"database": self.relation_data.database} + + if self.relation_data.extra_user_roles: + event_data["extra-user-roles"] = self.relation_data.extra_user_roles + + # set external-node-connectivity field + if self.relation_data.external_node_connectivity: + event_data["external-node-connectivity"] = "true" + + self.relation_data.update_relation_data(event.relation.id, event_data) def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the database relation has changed.""" @@ -1944,12 +2842,12 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: diff = self._diff(event) # Register all new secrets with their labels - if any(newval for newval in diff.added if self._is_secret_field(newval)): - self._register_secrets_to_relation(event.relation, diff.added) + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) # Check if the database is created # (the database charm shared the credentials). - secret_field_user = self._generate_secret_field_name(SecretGroup.USER) + secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) if ( "username" in diff.added and "password" in diff.added ) or secret_field_user in diff.added: @@ -1995,7 +2893,37 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: self._emit_aliased_event(event, "read_only_endpoints_changed") -# Kafka related events +class DatabaseRequires(DatabaseRequirerData, DatabaseRequirerEventHandlers): + """Provider-side of the database relations.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + database_name: str, + extra_user_roles: Optional[str] = None, + relations_aliases: Optional[List[str]] = None, + additional_secret_fields: Optional[List[str]] = [], + external_node_connectivity: bool = False, + ): + DatabaseRequirerData.__init__( + self, + charm.model, + relation_name, + database_name, + extra_user_roles, + relations_aliases, + additional_secret_fields, + external_node_connectivity, + ) + DatabaseRequirerEventHandlers.__init__(self, charm, self) + + +################################################################################ +# Charm-specific Relations Data and Events +################################################################################ + +# Kafka Events class KafkaProvidesEvent(RelationEvent): @@ -2088,29 +3016,11 @@ class KafkaRequiresEvents(CharmEvents): # Kafka Provides and Requires -class KafkaProvides(DataProvides): +class KafkaProviderData(ProviderData): """Provider-side of the Kafka relation.""" - on = KafkaProvidesEvents() # pyright: ignore [reportGeneralTypeIssues] - - def __init__(self, charm: CharmBase, relation_name: str) -> None: - super().__init__(charm, relation_name) - - def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: - """Event emitted when the relation has changed.""" - # Leader only - if not self.local_unit.is_leader(): - return - - # Check which data has changed to emit customs events. - diff = self._diff(event) - - # Emit a topic requested event if the setup key (topic name and optional - # extra user roles) was added to the relation databag by the application. - if "topic" in diff.added: - getattr(self.on, "topic_requested").emit( - event.relation, app=event.app, unit=event.unit - ) + def __init__(self, model: Model, relation_name: str) -> None: + super().__init__(model, relation_name) def set_topic(self, relation_id: int, topic: str) -> None: """Set topic name in the application relation databag. @@ -2149,14 +3059,47 @@ def set_zookeeper_uris(self, relation_id: int, zookeeper_uris: str) -> None: self.update_relation_data(relation_id, {"zookeeper-uris": zookeeper_uris}) -class KafkaRequires(DataRequires): - """Requires-side of the Kafka relation.""" +class KafkaProviderEventHandlers(EventHandlers): + """Provider-side of the Kafka relation.""" + + on = KafkaProvidesEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KafkaProviderData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Emit a topic requested event if the setup key (topic name and optional + # extra user roles) was added to the relation databag by the application. + if "topic" in diff.added: + getattr(self.on, "topic_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + +class KafkaProvides(KafkaProviderData, KafkaProviderEventHandlers): + """Provider-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + KafkaProviderData.__init__(self, charm.model, relation_name) + KafkaProviderEventHandlers.__init__(self, charm, self) - on = KafkaRequiresEvents() # pyright: ignore [reportGeneralTypeIssues] + +class KafkaRequirerData(RequirerData): + """Requirer-side of the Kafka relation.""" def __init__( self, - charm, + model: Model, relation_name: str, topic: str, extra_user_roles: Optional[str] = None, @@ -2164,9 +3107,7 @@ def __init__( additional_secret_fields: Optional[List[str]] = [], ): """Manager of Kafka client relations.""" - # super().__init__(charm, relation_name) - super().__init__(charm, relation_name, extra_user_roles, additional_secret_fields) - self.charm = charm + super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) self.topic = topic self.consumer_group_prefix = consumer_group_prefix or "" @@ -2182,20 +3123,34 @@ def topic(self, value): raise ValueError(f"Error on topic '{value}', cannot be a wildcard.") self._topic = value + +class KafkaRequirerEventHandlers(RequirerEventHandlers): + """Requires-side of the Kafka relation.""" + + on = KafkaRequiresEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KafkaRequirerData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: """Event emitted when the Kafka relation is created.""" super()._on_relation_created_event(event) - if not self.local_unit.is_leader(): + if not self.relation_data.local_unit.is_leader(): return # Sets topic, extra user roles, and "consumer-group-prefix" in the relation - relation_data = { - f: getattr(self, f.replace("-", "_"), "") - for f in ["consumer-group-prefix", "extra-user-roles", "topic"] - } + relation_data = {"topic": self.relation_data.topic} + + if self.relation_data.extra_user_roles: + relation_data["extra-user-roles"] = self.relation_data.extra_user_roles - self.update_relation_data(event.relation.id, relation_data) + if self.relation_data.consumer_group_prefix: + relation_data["consumer-group-prefix"] = self.relation_data.consumer_group_prefix + + self.relation_data.update_relation_data(event.relation.id, relation_data) def _on_secret_changed_event(self, event: SecretChangedEvent): """Event notifying about a new value of a secret.""" @@ -2210,10 +3165,10 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # (the Kafka charm shared the credentials). # Register all new secrets with their labels - if any(newval for newval in diff.added if self._is_secret_field(newval)): - self._register_secrets_to_relation(event.relation, diff.added) + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) - secret_field_user = self._generate_secret_field_name(SecretGroup.USER) + secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) if ( "username" in diff.added and "password" in diff.added ) or secret_field_user in diff.added: @@ -2236,6 +3191,30 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: return +class KafkaRequires(KafkaRequirerData, KafkaRequirerEventHandlers): + """Provider-side of the Kafka relation.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + topic: str, + extra_user_roles: Optional[str] = None, + consumer_group_prefix: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + ) -> None: + KafkaRequirerData.__init__( + self, + charm.model, + relation_name, + topic, + extra_user_roles, + consumer_group_prefix, + additional_secret_fields, + ) + KafkaRequirerEventHandlers.__init__(self, charm, self) + + # Opensearch related events @@ -2286,28 +3265,11 @@ class OpenSearchRequiresEvents(CharmEvents): # OpenSearch Provides and Requires Objects -class OpenSearchProvides(DataProvides): +class OpenSearchProvidesData(ProviderData): """Provider-side of the OpenSearch relation.""" - on = OpenSearchProvidesEvents() # pyright: ignore[reportGeneralTypeIssues] - - def __init__(self, charm: CharmBase, relation_name: str) -> None: - super().__init__(charm, relation_name) - - def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: - """Event emitted when the relation has changed.""" - # Leader only - if not self.local_unit.is_leader(): - return - # Check which data has changed to emit customs events. - diff = self._diff(event) - - # Emit an index requested event if the setup key (index name and optional extra user roles) - # have been added to the relation databag by the application. - if "index" in diff.added: - getattr(self.on, "index_requested").emit( - event.relation, app=event.app, unit=event.unit - ) + def __init__(self, model: Model, relation_name: str) -> None: + super().__init__(model, relation_name) def set_index(self, relation_id: int, index: str) -> None: """Set the index in the application relation databag. @@ -2339,45 +3301,87 @@ def set_version(self, relation_id: int, version: str) -> None: self.update_relation_data(relation_id, {"version": version}) -class OpenSearchRequires(DataRequires): - """Requires-side of the OpenSearch relation.""" +class OpenSearchProvidesEventHandlers(EventHandlers): + """Provider-side of the OpenSearch relation.""" + + on = OpenSearchProvidesEvents() # pyright: ignore[reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: OpenSearchProvidesData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Emit an index requested event if the setup key (index name and optional extra user roles) + # have been added to the relation databag by the application. + if "index" in diff.added: + getattr(self.on, "index_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + +class OpenSearchProvides(OpenSearchProvidesData, OpenSearchProvidesEventHandlers): + """Provider-side of the OpenSearch relation.""" + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + OpenSearchProvidesData.__init__(self, charm.model, relation_name) + OpenSearchProvidesEventHandlers.__init__(self, charm, self) - on = OpenSearchRequiresEvents() # pyright: ignore[reportGeneralTypeIssues] + +class OpenSearchRequiresData(RequirerData): + """Requires data side of the OpenSearch relation.""" def __init__( self, - charm, + model: Model, relation_name: str, index: str, extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], ): """Manager of OpenSearch client relations.""" - super().__init__(charm, relation_name, extra_user_roles, additional_secret_fields) - self.charm = charm + super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) self.index = index + +class OpenSearchRequiresEventHandlers(RequirerEventHandlers): + """Requires events side of the OpenSearch relation.""" + + on = OpenSearchRequiresEvents() # pyright: ignore[reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: OpenSearchRequiresData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: """Event emitted when the OpenSearch relation is created.""" super()._on_relation_created_event(event) - if not self.local_unit.is_leader(): + if not self.relation_data.local_unit.is_leader(): return # Sets both index and extra user roles in the relation if the roles are provided. # Otherwise, sets only the index. - data = {"index": self.index} - if self.extra_user_roles: - data["extra-user-roles"] = self.extra_user_roles + data = {"index": self.relation_data.index} + if self.relation_data.extra_user_roles: + data["extra-user-roles"] = self.relation_data.extra_user_roles - self.update_relation_data(event.relation.id, data) + self.relation_data.update_relation_data(event.relation.id, data) def _on_secret_changed_event(self, event: SecretChangedEvent): """Event notifying about a new value of a secret.""" if not event.secret.label: return - relation = self._relation_from_secret_label(event.secret.label) + relation = self.relation_data._relation_from_secret_label(event.secret.label) if not relation: logging.info( f"Received secret {event.secret.label} but couldn't parse, seems irrelevant" @@ -2406,11 +3410,11 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: diff = self._diff(event) # Register all new secrets with their labels - if any(newval for newval in diff.added if self._is_secret_field(newval)): - self._register_secrets_to_relation(event.relation, diff.added) + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) - secret_field_user = self._generate_secret_field_name(SecretGroup.USER) - secret_field_tls = self._generate_secret_field_name(SecretGroup.TLS) + secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) + secret_field_tls = self.relation_data._generate_secret_field_name(SECRET_GROUPS.TLS) updates = {"username", "password", "tls", "tls-ca", secret_field_user, secret_field_tls} if len(set(diff._asdict().keys()) - updates) < len(diff): logger.info("authentication updated at: %s", datetime.now()) @@ -2440,3 +3444,25 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: event.relation, app=event.app, unit=event.unit ) # here check if this is the right design return + + +class OpenSearchRequires(OpenSearchRequiresData, OpenSearchRequiresEventHandlers): + """Requires-side of the OpenSearch relation.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + index: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + ) -> None: + OpenSearchRequiresData.__init__( + self, + charm.model, + relation_name, + index, + extra_user_roles, + additional_secret_fields, + ) + OpenSearchRequiresEventHandlers.__init__(self, charm, self) diff --git a/poetry.lock b/poetry.lock index 317f2980..da1bda01 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "appnope" @@ -363,13 +363,13 @@ files = [ [[package]] name = "cosl" -version = "0.0.10" +version = "0.0.11" description = "Utils for COS Lite charms" optional = false python-versions = ">=3.8" files = [ - {file = "cosl-0.0.10-py3-none-any.whl", hash = "sha256:9bbfb85917460075780cb8f94774c41dc2a8890ae06d35ad9b4c78d2a20fe803"}, - {file = "cosl-0.0.10.tar.gz", hash = "sha256:fe45ef7086a4948f5f9546e620b001822104b7c89a4a34f297d4d1acc117356f"}, + {file = "cosl-0.0.11-py3-none-any.whl", hash = "sha256:46d78d6441ba628bae386cd8c10b8144558ab208115522020e7858f97837988d"}, + {file = "cosl-0.0.11.tar.gz", hash = "sha256:15cac6ed20b65e9d33cda3c3da32e299c82f9feea64e393448cd3d3cf2bef32a"}, ] [package.dependencies] @@ -379,63 +379,63 @@ typing-extensions = "*" [[package]] name = "coverage" -version = "7.4.4" +version = "7.5.0" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0be5efd5127542ef31f165de269f77560d6cdef525fffa446de6f7e9186cfb2"}, - {file = "coverage-7.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ccd341521be3d1b3daeb41960ae94a5e87abe2f46f17224ba5d6f2b8398016cf"}, - {file = "coverage-7.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fa497a8ab37784fbb20ab699c246053ac294d13fc7eb40ec007a5043ec91f8"}, - {file = "coverage-7.4.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b1a93009cb80730c9bca5d6d4665494b725b6e8e157c1cb7f2db5b4b122ea562"}, - {file = "coverage-7.4.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:690db6517f09336559dc0b5f55342df62370a48f5469fabf502db2c6d1cffcd2"}, - {file = "coverage-7.4.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:09c3255458533cb76ef55da8cc49ffab9e33f083739c8bd4f58e79fecfe288f7"}, - {file = "coverage-7.4.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8ce1415194b4a6bd0cdcc3a1dfbf58b63f910dcb7330fe15bdff542c56949f87"}, - {file = "coverage-7.4.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b91cbc4b195444e7e258ba27ac33769c41b94967919f10037e6355e998af255c"}, - {file = "coverage-7.4.4-cp310-cp310-win32.whl", hash = "sha256:598825b51b81c808cb6f078dcb972f96af96b078faa47af7dfcdf282835baa8d"}, - {file = "coverage-7.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:09ef9199ed6653989ebbcaacc9b62b514bb63ea2f90256e71fea3ed74bd8ff6f"}, - {file = "coverage-7.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0f9f50e7ef2a71e2fae92774c99170eb8304e3fdf9c8c3c7ae9bab3e7229c5cf"}, - {file = "coverage-7.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:623512f8ba53c422fcfb2ce68362c97945095b864cda94a92edbaf5994201083"}, - {file = "coverage-7.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0513b9508b93da4e1716744ef6ebc507aff016ba115ffe8ecff744d1322a7b63"}, - {file = "coverage-7.4.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40209e141059b9370a2657c9b15607815359ab3ef9918f0196b6fccce8d3230f"}, - {file = "coverage-7.4.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a2b2b78c78293782fd3767d53e6474582f62443d0504b1554370bde86cc8227"}, - {file = "coverage-7.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:73bfb9c09951125d06ee473bed216e2c3742f530fc5acc1383883125de76d9cd"}, - {file = "coverage-7.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1f384c3cc76aeedce208643697fb3e8437604b512255de6d18dae3f27655a384"}, - {file = "coverage-7.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:54eb8d1bf7cacfbf2a3186019bcf01d11c666bd495ed18717162f7eb1e9dd00b"}, - {file = "coverage-7.4.4-cp311-cp311-win32.whl", hash = "sha256:cac99918c7bba15302a2d81f0312c08054a3359eaa1929c7e4b26ebe41e9b286"}, - {file = "coverage-7.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:b14706df8b2de49869ae03a5ccbc211f4041750cd4a66f698df89d44f4bd30ec"}, - {file = "coverage-7.4.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:201bef2eea65e0e9c56343115ba3814e896afe6d36ffd37bab783261db430f76"}, - {file = "coverage-7.4.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:41c9c5f3de16b903b610d09650e5e27adbfa7f500302718c9ffd1c12cf9d6818"}, - {file = "coverage-7.4.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d898fe162d26929b5960e4e138651f7427048e72c853607f2b200909794ed978"}, - {file = "coverage-7.4.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ea79bb50e805cd6ac058dfa3b5c8f6c040cb87fe83de10845857f5535d1db70"}, - {file = "coverage-7.4.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce4b94265ca988c3f8e479e741693d143026632672e3ff924f25fab50518dd51"}, - {file = "coverage-7.4.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:00838a35b882694afda09f85e469c96367daa3f3f2b097d846a7216993d37f4c"}, - {file = "coverage-7.4.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:fdfafb32984684eb03c2d83e1e51f64f0906b11e64482df3c5db936ce3839d48"}, - {file = "coverage-7.4.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:69eb372f7e2ece89f14751fbcbe470295d73ed41ecd37ca36ed2eb47512a6ab9"}, - {file = "coverage-7.4.4-cp312-cp312-win32.whl", hash = "sha256:137eb07173141545e07403cca94ab625cc1cc6bc4c1e97b6e3846270e7e1fea0"}, - {file = "coverage-7.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:d71eec7d83298f1af3326ce0ff1d0ea83c7cb98f72b577097f9083b20bdaf05e"}, - {file = "coverage-7.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d5ae728ff3b5401cc320d792866987e7e7e880e6ebd24433b70a33b643bb0384"}, - {file = "coverage-7.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cc4f1358cb0c78edef3ed237ef2c86056206bb8d9140e73b6b89fbcfcbdd40e1"}, - {file = "coverage-7.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8130a2aa2acb8788e0b56938786c33c7c98562697bf9f4c7d6e8e5e3a0501e4a"}, - {file = "coverage-7.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf271892d13e43bc2b51e6908ec9a6a5094a4df1d8af0bfc360088ee6c684409"}, - {file = "coverage-7.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4cdc86d54b5da0df6d3d3a2f0b710949286094c3a6700c21e9015932b81447e"}, - {file = "coverage-7.4.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ae71e7ddb7a413dd60052e90528f2f65270aad4b509563af6d03d53e979feafd"}, - {file = "coverage-7.4.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:38dd60d7bf242c4ed5b38e094baf6401faa114fc09e9e6632374388a404f98e7"}, - {file = "coverage-7.4.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aa5b1c1bfc28384f1f53b69a023d789f72b2e0ab1b3787aae16992a7ca21056c"}, - {file = "coverage-7.4.4-cp38-cp38-win32.whl", hash = "sha256:dfa8fe35a0bb90382837b238fff375de15f0dcdb9ae68ff85f7a63649c98527e"}, - {file = "coverage-7.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:b2991665420a803495e0b90a79233c1433d6ed77ef282e8e152a324bbbc5e0c8"}, - {file = "coverage-7.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3b799445b9f7ee8bf299cfaed6f5b226c0037b74886a4e11515e569b36fe310d"}, - {file = "coverage-7.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b4d33f418f46362995f1e9d4f3a35a1b6322cb959c31d88ae56b0298e1c22357"}, - {file = "coverage-7.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aadacf9a2f407a4688d700e4ebab33a7e2e408f2ca04dbf4aef17585389eff3e"}, - {file = "coverage-7.4.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c95949560050d04d46b919301826525597f07b33beba6187d04fa64d47ac82e"}, - {file = "coverage-7.4.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff7687ca3d7028d8a5f0ebae95a6e4827c5616b31a4ee1192bdfde697db110d4"}, - {file = "coverage-7.4.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5fc1de20b2d4a061b3df27ab9b7c7111e9a710f10dc2b84d33a4ab25065994ec"}, - {file = "coverage-7.4.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c74880fc64d4958159fbd537a091d2a585448a8f8508bf248d72112723974cbd"}, - {file = "coverage-7.4.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:742a76a12aa45b44d236815d282b03cfb1de3b4323f3e4ec933acfae08e54ade"}, - {file = "coverage-7.4.4-cp39-cp39-win32.whl", hash = "sha256:d89d7b2974cae412400e88f35d86af72208e1ede1a541954af5d944a8ba46c57"}, - {file = "coverage-7.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:9ca28a302acb19b6af89e90f33ee3e1906961f94b54ea37de6737b7ca9d8827c"}, - {file = "coverage-7.4.4-pp38.pp39.pp310-none-any.whl", hash = "sha256:b2c5edc4ac10a7ef6605a966c58929ec6c1bd0917fb8c15cb3363f65aa40e677"}, - {file = "coverage-7.4.4.tar.gz", hash = "sha256:c901df83d097649e257e803be22592aedfd5182f07b3cc87d640bbb9afd50f49"}, + {file = "coverage-7.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:432949a32c3e3f820af808db1833d6d1631664d53dd3ce487aa25d574e18ad1c"}, + {file = "coverage-7.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2bd7065249703cbeb6d4ce679c734bef0ee69baa7bff9724361ada04a15b7e3b"}, + {file = "coverage-7.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbfe6389c5522b99768a93d89aca52ef92310a96b99782973b9d11e80511f932"}, + {file = "coverage-7.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39793731182c4be939b4be0cdecde074b833f6171313cf53481f869937129ed3"}, + {file = "coverage-7.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85a5dbe1ba1bf38d6c63b6d2c42132d45cbee6d9f0c51b52c59aa4afba057517"}, + {file = "coverage-7.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:357754dcdfd811462a725e7501a9b4556388e8ecf66e79df6f4b988fa3d0b39a"}, + {file = "coverage-7.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a81eb64feded34f40c8986869a2f764f0fe2db58c0530d3a4afbcde50f314880"}, + {file = "coverage-7.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:51431d0abbed3a868e967f8257c5faf283d41ec882f58413cf295a389bb22e58"}, + {file = "coverage-7.5.0-cp310-cp310-win32.whl", hash = "sha256:f609ebcb0242d84b7adeee2b06c11a2ddaec5464d21888b2c8255f5fd6a98ae4"}, + {file = "coverage-7.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:6782cd6216fab5a83216cc39f13ebe30adfac2fa72688c5a4d8d180cd52e8f6a"}, + {file = "coverage-7.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e768d870801f68c74c2b669fc909839660180c366501d4cc4b87efd6b0eee375"}, + {file = "coverage-7.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:84921b10aeb2dd453247fd10de22907984eaf80901b578a5cf0bb1e279a587cb"}, + {file = "coverage-7.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:710c62b6e35a9a766b99b15cdc56d5aeda0914edae8bb467e9c355f75d14ee95"}, + {file = "coverage-7.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c379cdd3efc0658e652a14112d51a7668f6bfca7445c5a10dee7eabecabba19d"}, + {file = "coverage-7.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fea9d3ca80bcf17edb2c08a4704259dadac196fe5e9274067e7a20511fad1743"}, + {file = "coverage-7.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:41327143c5b1d715f5f98a397608f90ab9ebba606ae4e6f3389c2145410c52b1"}, + {file = "coverage-7.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:565b2e82d0968c977e0b0f7cbf25fd06d78d4856289abc79694c8edcce6eb2de"}, + {file = "coverage-7.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cf3539007202ebfe03923128fedfdd245db5860a36810136ad95a564a2fdffff"}, + {file = "coverage-7.5.0-cp311-cp311-win32.whl", hash = "sha256:bf0b4b8d9caa8d64df838e0f8dcf68fb570c5733b726d1494b87f3da85db3a2d"}, + {file = "coverage-7.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:9c6384cc90e37cfb60435bbbe0488444e54b98700f727f16f64d8bfda0b84656"}, + {file = "coverage-7.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fed7a72d54bd52f4aeb6c6e951f363903bd7d70bc1cad64dd1f087980d309ab9"}, + {file = "coverage-7.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cbe6581fcff7c8e262eb574244f81f5faaea539e712a058e6707a9d272fe5b64"}, + {file = "coverage-7.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad97ec0da94b378e593ef532b980c15e377df9b9608c7c6da3506953182398af"}, + {file = "coverage-7.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd4bacd62aa2f1a1627352fe68885d6ee694bdaebb16038b6e680f2924a9b2cc"}, + {file = "coverage-7.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:adf032b6c105881f9d77fa17d9eebe0ad1f9bfb2ad25777811f97c5362aa07f2"}, + {file = "coverage-7.5.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ba01d9ba112b55bfa4b24808ec431197bb34f09f66f7cb4fd0258ff9d3711b1"}, + {file = "coverage-7.5.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f0bfe42523893c188e9616d853c47685e1c575fe25f737adf473d0405dcfa7eb"}, + {file = "coverage-7.5.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a9a7ef30a1b02547c1b23fa9a5564f03c9982fc71eb2ecb7f98c96d7a0db5cf2"}, + {file = "coverage-7.5.0-cp312-cp312-win32.whl", hash = "sha256:3c2b77f295edb9fcdb6a250f83e6481c679335ca7e6e4a955e4290350f2d22a4"}, + {file = "coverage-7.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:427e1e627b0963ac02d7c8730ca6d935df10280d230508c0ba059505e9233475"}, + {file = "coverage-7.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9dd88fce54abbdbf4c42fb1fea0e498973d07816f24c0e27a1ecaf91883ce69e"}, + {file = "coverage-7.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a898c11dca8f8c97b467138004a30133974aacd572818c383596f8d5b2eb04a9"}, + {file = "coverage-7.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07dfdd492d645eea1bd70fb1d6febdcf47db178b0d99161d8e4eed18e7f62fe7"}, + {file = "coverage-7.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3d117890b6eee85887b1eed41eefe2e598ad6e40523d9f94c4c4b213258e4a4"}, + {file = "coverage-7.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6afd2e84e7da40fe23ca588379f815fb6dbbb1b757c883935ed11647205111cb"}, + {file = "coverage-7.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a9960dd1891b2ddf13a7fe45339cd59ecee3abb6b8326d8b932d0c5da208104f"}, + {file = "coverage-7.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ced268e82af993d7801a9db2dbc1d2322e786c5dc76295d8e89473d46c6b84d4"}, + {file = "coverage-7.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e7c211f25777746d468d76f11719e64acb40eed410d81c26cefac641975beb88"}, + {file = "coverage-7.5.0-cp38-cp38-win32.whl", hash = "sha256:262fffc1f6c1a26125d5d573e1ec379285a3723363f3bd9c83923c9593a2ac25"}, + {file = "coverage-7.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:eed462b4541c540d63ab57b3fc69e7d8c84d5957668854ee4e408b50e92ce26a"}, + {file = "coverage-7.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d0194d654e360b3e6cc9b774e83235bae6b9b2cac3be09040880bb0e8a88f4a1"}, + {file = "coverage-7.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:33c020d3322662e74bc507fb11488773a96894aa82a622c35a5a28673c0c26f5"}, + {file = "coverage-7.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbdf2cae14a06827bec50bd58e49249452d211d9caddd8bd80e35b53cb04631"}, + {file = "coverage-7.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3235d7c781232e525b0761730e052388a01548bd7f67d0067a253887c6e8df46"}, + {file = "coverage-7.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2de4e546f0ec4b2787d625e0b16b78e99c3e21bc1722b4977c0dddf11ca84e"}, + {file = "coverage-7.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4d0e206259b73af35c4ec1319fd04003776e11e859936658cb6ceffdeba0f5be"}, + {file = "coverage-7.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2055c4fb9a6ff624253d432aa471a37202cd8f458c033d6d989be4499aed037b"}, + {file = "coverage-7.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:075299460948cd12722a970c7eae43d25d37989da682997687b34ae6b87c0ef0"}, + {file = "coverage-7.5.0-cp39-cp39-win32.whl", hash = "sha256:280132aada3bc2f0fac939a5771db4fbb84f245cb35b94fae4994d4c1f80dae7"}, + {file = "coverage-7.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:c58536f6892559e030e6924896a44098bc1290663ea12532c78cef71d0df8493"}, + {file = "coverage-7.5.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:2b57780b51084d5223eee7b59f0d4911c31c16ee5aa12737c7a02455829ff067"}, + {file = "coverage-7.5.0.tar.gz", hash = "sha256:cf62d17310f34084c59c01e027259076479128d11e4661bb6c9acb38c5e19bb8"}, ] [package.dependencies] @@ -511,13 +511,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.2.0" +version = "1.2.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, - {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, ] [package.extras] @@ -562,13 +562,13 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "hvac" -version = "2.1.0" +version = "2.2.0" description = "HashiCorp Vault API client" optional = false -python-versions = ">=3.8,<4.0" +python-versions = "<4.0,>=3.8" files = [ - {file = "hvac-2.1.0-py3-none-any.whl", hash = "sha256:73bc91e58c3fc7c6b8107cdaca9cb71fa0a893dfd80ffbc1c14e20f24c0c29d7"}, - {file = "hvac-2.1.0.tar.gz", hash = "sha256:b48bcda11a4ab0a7b6c47232c7ba7c87fda318ae2d4a7662800c465a78742894"}, + {file = "hvac-2.2.0-py3-none-any.whl", hash = "sha256:f287a19940c6fc518c723f8276cc9927f7400734303ee5872ac2e84539466d8d"}, + {file = "hvac-2.2.0.tar.gz", hash = "sha256:e4b0248c5672cb9a6f5974e7c8f5271a09c6c663cbf8ab11733a227f3d2db2c2"}, ] [package.dependencies] @@ -579,13 +579,13 @@ parser = ["pyhcl (>=0.4.4,<0.5.0)"] [[package]] name = "idna" -version = "3.6" +version = "3.7" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, - {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] [[package]] @@ -710,13 +710,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "jsonschema" -version = "4.21.1" +version = "4.22.0" description = "An implementation of JSON Schema validation for Python" optional = false python-versions = ">=3.8" files = [ - {file = "jsonschema-4.21.1-py3-none-any.whl", hash = "sha256:7996507afae316306f9e2290407761157c6f78002dcf7419acb99822143d1c6f"}, - {file = "jsonschema-4.21.1.tar.gz", hash = "sha256:85727c00279f5fa6bedbe6238d2aa6403bedd8b4864ab11207d07df3cc1b2ee5"}, + {file = "jsonschema-4.22.0-py3-none-any.whl", hash = "sha256:ff4cfd6b1367a40e7bc6411caec72effadd3db0bbe5017de188f2d6108335802"}, + {file = "jsonschema-4.22.0.tar.gz", hash = "sha256:5b22d434a45935119af990552c862e5d6d564e8f6601206b305a61fdf661a2b7"}, ] [package.dependencies] @@ -920,13 +920,13 @@ files = [ [[package]] name = "matplotlib-inline" -version = "0.1.6" +version = "0.1.7" description = "Inline Matplotlib backend for Jupyter" optional = false -python-versions = ">=3.5" +python-versions = ">=3.8" files = [ - {file = "matplotlib-inline-0.1.6.tar.gz", hash = "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"}, - {file = "matplotlib_inline-0.1.6-py3-none-any.whl", hash = "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311"}, + {file = "matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"}, + {file = "matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90"}, ] [package.dependencies] @@ -975,13 +975,13 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "ops" -version = "2.12.0" +version = "2.13.0" description = "The Python library behind great charms" optional = false python-versions = ">=3.8" files = [ - {file = "ops-2.12.0-py3-none-any.whl", hash = "sha256:b6f7db8aa2886351d0a2527f0df6c8a34e0d9cf90ddfbb91e734f73259df8ddf"}, - {file = "ops-2.12.0.tar.gz", hash = "sha256:7d88522914728caa13aaf1689637f8b573eaf5d38b7f2b8cf135406ee6ef0fc3"}, + {file = "ops-2.13.0-py3-none-any.whl", hash = "sha256:edebef03841d727a9b8bd9ee3f52c5b94070fd748641a0927b51f6fe3a887365"}, + {file = "ops-2.13.0.tar.gz", hash = "sha256:106deec8c18a6dbf7fa3e6fe6e288784b1da8cb626b5265f6c4b959e10877272"}, ] [package.dependencies] @@ -1025,18 +1025,18 @@ invoke = ["invoke (>=2.0)"] [[package]] name = "parso" -version = "0.8.3" +version = "0.8.4" description = "A Python Parser" optional = false python-versions = ">=3.6" files = [ - {file = "parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"}, - {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, + {file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"}, + {file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"}, ] [package.extras] -qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] -testing = ["docopt", "pytest (<6.0.0)"] +qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] +testing = ["docopt", "pytest"] [[package]] name = "pathspec" @@ -1087,28 +1087,29 @@ files = [ [[package]] name = "platformdirs" -version = "4.2.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +version = "4.2.1" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, - {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, + {file = "platformdirs-4.2.1-py3-none-any.whl", hash = "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"}, + {file = "platformdirs-4.2.1.tar.gz", hash = "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf"}, ] [package.extras] docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] [[package]] name = "pluggy" -version = "1.4.0" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, - {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [package.extras] @@ -1226,47 +1227,47 @@ files = [ [[package]] name = "pydantic" -version = "1.10.14" +version = "1.10.15" description = "Data validation and settings management using python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, - {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, - {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, - {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, - {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, - {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, - {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, - {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, - {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, - {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, - {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, - {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, - {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, - {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, - {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, - {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, - {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, - {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, - {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, - {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, - {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, - {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, - {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, - {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, - {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, - {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, - {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, - {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, - {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, - {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, - {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, - {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, - {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, - {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, - {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, - {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, + {file = "pydantic-1.10.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:22ed12ee588b1df028a2aa5d66f07bf8f8b4c8579c2e96d5a9c1f96b77f3bb55"}, + {file = "pydantic-1.10.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:75279d3cac98186b6ebc2597b06bcbc7244744f6b0b44a23e4ef01e5683cc0d2"}, + {file = "pydantic-1.10.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50f1666a9940d3d68683c9d96e39640f709d7a72ff8702987dab1761036206bb"}, + {file = "pydantic-1.10.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82790d4753ee5d00739d6cb5cf56bceb186d9d6ce134aca3ba7befb1eedbc2c8"}, + {file = "pydantic-1.10.15-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d207d5b87f6cbefbdb1198154292faee8017d7495a54ae58db06762004500d00"}, + {file = "pydantic-1.10.15-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e49db944fad339b2ccb80128ffd3f8af076f9f287197a480bf1e4ca053a866f0"}, + {file = "pydantic-1.10.15-cp310-cp310-win_amd64.whl", hash = "sha256:d3b5c4cbd0c9cb61bbbb19ce335e1f8ab87a811f6d589ed52b0254cf585d709c"}, + {file = "pydantic-1.10.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c3d5731a120752248844676bf92f25a12f6e45425e63ce22e0849297a093b5b0"}, + {file = "pydantic-1.10.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c365ad9c394f9eeffcb30a82f4246c0006417f03a7c0f8315d6211f25f7cb654"}, + {file = "pydantic-1.10.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3287e1614393119c67bd4404f46e33ae3be3ed4cd10360b48d0a4459f420c6a3"}, + {file = "pydantic-1.10.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be51dd2c8596b25fe43c0a4a59c2bee4f18d88efb8031188f9e7ddc6b469cf44"}, + {file = "pydantic-1.10.15-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6a51a1dd4aa7b3f1317f65493a182d3cff708385327c1c82c81e4a9d6d65b2e4"}, + {file = "pydantic-1.10.15-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4e316e54b5775d1eb59187f9290aeb38acf620e10f7fd2f776d97bb788199e53"}, + {file = "pydantic-1.10.15-cp311-cp311-win_amd64.whl", hash = "sha256:0d142fa1b8f2f0ae11ddd5e3e317dcac060b951d605fda26ca9b234b92214986"}, + {file = "pydantic-1.10.15-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7ea210336b891f5ea334f8fc9f8f862b87acd5d4a0cbc9e3e208e7aa1775dabf"}, + {file = "pydantic-1.10.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3453685ccd7140715e05f2193d64030101eaad26076fad4e246c1cc97e1bb30d"}, + {file = "pydantic-1.10.15-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bea1f03b8d4e8e86702c918ccfd5d947ac268f0f0cc6ed71782e4b09353b26f"}, + {file = "pydantic-1.10.15-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:005655cabc29081de8243126e036f2065bd7ea5b9dff95fde6d2c642d39755de"}, + {file = "pydantic-1.10.15-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:af9850d98fc21e5bc24ea9e35dd80a29faf6462c608728a110c0a30b595e58b7"}, + {file = "pydantic-1.10.15-cp37-cp37m-win_amd64.whl", hash = "sha256:d31ee5b14a82c9afe2bd26aaa405293d4237d0591527d9129ce36e58f19f95c1"}, + {file = "pydantic-1.10.15-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5e09c19df304b8123938dc3c53d3d3be6ec74b9d7d0d80f4f4b5432ae16c2022"}, + {file = "pydantic-1.10.15-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7ac9237cd62947db00a0d16acf2f3e00d1ae9d3bd602b9c415f93e7a9fc10528"}, + {file = "pydantic-1.10.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:584f2d4c98ffec420e02305cf675857bae03c9d617fcfdc34946b1160213a948"}, + {file = "pydantic-1.10.15-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bbc6989fad0c030bd70a0b6f626f98a862224bc2b1e36bfc531ea2facc0a340c"}, + {file = "pydantic-1.10.15-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d573082c6ef99336f2cb5b667b781d2f776d4af311574fb53d908517ba523c22"}, + {file = "pydantic-1.10.15-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6bd7030c9abc80134087d8b6e7aa957e43d35714daa116aced57269a445b8f7b"}, + {file = "pydantic-1.10.15-cp38-cp38-win_amd64.whl", hash = "sha256:3350f527bb04138f8aff932dc828f154847fbdc7a1a44c240fbfff1b57f49a12"}, + {file = "pydantic-1.10.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:51d405b42f1b86703555797270e4970a9f9bd7953f3990142e69d1037f9d9e51"}, + {file = "pydantic-1.10.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a980a77c52723b0dc56640ced396b73a024d4b74f02bcb2d21dbbac1debbe9d0"}, + {file = "pydantic-1.10.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67f1a1fb467d3f49e1708a3f632b11c69fccb4e748a325d5a491ddc7b5d22383"}, + {file = "pydantic-1.10.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:676ed48f2c5bbad835f1a8ed8a6d44c1cd5a21121116d2ac40bd1cd3619746ed"}, + {file = "pydantic-1.10.15-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:92229f73400b80c13afcd050687f4d7e88de9234d74b27e6728aa689abcf58cc"}, + {file = "pydantic-1.10.15-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2746189100c646682eff0bce95efa7d2e203420d8e1c613dc0c6b4c1d9c1fde4"}, + {file = "pydantic-1.10.15-cp39-cp39-win_amd64.whl", hash = "sha256:394f08750bd8eaad714718812e7fab615f873b3cdd0b9d84e76e51ef3b50b6b7"}, + {file = "pydantic-1.10.15-py3-none-any.whl", hash = "sha256:28e552a060ba2740d0d2aabe35162652c1459a0b9069fe0db7f4ee0e18e74d58"}, + {file = "pydantic-1.10.15.tar.gz", hash = "sha256:ca832e124eda231a60a041da4f013e3ff24949d94a01154b137fc2f2a43c3ffb"}, ] [package.dependencies] @@ -1348,13 +1349,13 @@ pytz = "*" [[package]] name = "pyright" -version = "1.1.356" +version = "1.1.360" description = "Command line wrapper for pyright" optional = false python-versions = ">=3.7" files = [ - {file = "pyright-1.1.356-py3-none-any.whl", hash = "sha256:a101b0f375f93d7082f9046cfaa7ba15b7cf8e1939ace45e984c351f6e8feb99"}, - {file = "pyright-1.1.356.tar.gz", hash = "sha256:f05b8b29d06b96ed4a0885dad5a31d9dff691ca12b2f658249f583d5f2754021"}, + {file = "pyright-1.1.360-py3-none-any.whl", hash = "sha256:7637f75451ac968b7cf1f8c51cfefb6d60ac7d086eb845364bc8ac03a026efd7"}, + {file = "pyright-1.1.360.tar.gz", hash = "sha256:784ddcda9745e9f5610483d7b963e9aa8d4f50d7755a9dffb28ccbeb27adce32"}, ] [package.dependencies] @@ -1366,13 +1367,13 @@ dev = ["twine (>=3.4.1)"] [[package]] name = "pytest" -version = "8.1.1" +version = "8.2.0" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.1.1-py3-none-any.whl", hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7"}, - {file = "pytest-8.1.1.tar.gz", hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044"}, + {file = "pytest-8.2.0-py3-none-any.whl", hash = "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233"}, + {file = "pytest-8.2.0.tar.gz", hash = "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f"}, ] [package.dependencies] @@ -1380,21 +1381,21 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=1.4,<2.0" +pluggy = ">=1.5,<2.0" tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-asyncio" -version = "0.21.1" +version = "0.21.2" description = "Pytest support for asyncio" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-asyncio-0.21.1.tar.gz", hash = "sha256:40a7eae6dded22c7b604986855ea48400ab15b069ae38116e8c01238e9eeb64d"}, - {file = "pytest_asyncio-0.21.1-py3-none-any.whl", hash = "sha256:8666c1c8ac02631d7c51ba282e0c69a8a452b211ffedf2599099845da5c5c37b"}, + {file = "pytest_asyncio-0.21.2-py3-none-any.whl", hash = "sha256:ab664c88bb7998f711d8039cacd4884da6430886ae8bbd4eded552ed2004f16b"}, + {file = "pytest_asyncio-0.21.2.tar.gz", hash = "sha256:d67738fc232b94b326b9d060750beb16e0074210b98dd8b58a5239fa2a154f45"}, ] [package.dependencies] @@ -1423,13 +1424,13 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] [[package]] name = "pytest-operator" -version = "0.34.0" +version = "0.35.0" description = "Fixtures for Operators" optional = false python-versions = "*" files = [ - {file = "pytest-operator-0.34.0.tar.gz", hash = "sha256:b2d85c666436fae41e8e8f914e12c07362c473caa0f325c58e1270b00fd4fca4"}, - {file = "pytest_operator-0.34.0-py3-none-any.whl", hash = "sha256:a3534ef376c5931beb04859359f18a4477001e14ed664459fb148cfafaffb943"}, + {file = "pytest-operator-0.35.0.tar.gz", hash = "sha256:ed963dc013fc576e218081e95197926b7c98116c1fb5ab234269cf72e0746d5b"}, + {file = "pytest_operator-0.35.0-py3-none-any.whl", hash = "sha256:026715faba7a0d725ca386fe05a45cfc73746293d8d755be6d2a67ca252267f5"}, ] [package.dependencies] @@ -1509,7 +1510,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -1546,13 +1546,13 @@ files = [ [[package]] name = "referencing" -version = "0.34.0" +version = "0.35.0" description = "JSON Referencing + Python" optional = false python-versions = ">=3.8" files = [ - {file = "referencing-0.34.0-py3-none-any.whl", hash = "sha256:d53ae300ceddd3169f1ffa9caf2cb7b769e92657e4fafb23d34b93679116dfd4"}, - {file = "referencing-0.34.0.tar.gz", hash = "sha256:5773bd84ef41799a5a8ca72dc34590c041eb01bf9aa02632b4a973fb0181a844"}, + {file = "referencing-0.35.0-py3-none-any.whl", hash = "sha256:8080727b30e364e5783152903672df9b6b091c926a146a759080b62ca3126cd6"}, + {file = "referencing-0.35.0.tar.gz", hash = "sha256:191e936b0c696d0af17ad7430a3dc68e88bc11be6514f4757dc890f04ab05889"}, ] [package.dependencies] @@ -1722,44 +1722,44 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.3.5" +version = "0.4.2" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.3.5-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:aef5bd3b89e657007e1be6b16553c8813b221ff6d92c7526b7e0227450981eac"}, - {file = "ruff-0.3.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:89b1e92b3bd9fca249153a97d23f29bed3992cff414b222fcd361d763fc53f12"}, - {file = "ruff-0.3.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e55771559c89272c3ebab23326dc23e7f813e492052391fe7950c1a5a139d89"}, - {file = "ruff-0.3.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dabc62195bf54b8a7876add6e789caae0268f34582333cda340497c886111c39"}, - {file = "ruff-0.3.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a05f3793ba25f194f395578579c546ca5d83e0195f992edc32e5907d142bfa3"}, - {file = "ruff-0.3.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:dfd3504e881082959b4160ab02f7a205f0fadc0a9619cc481982b6837b2fd4c0"}, - {file = "ruff-0.3.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87258e0d4b04046cf1d6cc1c56fadbf7a880cc3de1f7294938e923234cf9e498"}, - {file = "ruff-0.3.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:712e71283fc7d9f95047ed5f793bc019b0b0a29849b14664a60fd66c23b96da1"}, - {file = "ruff-0.3.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a532a90b4a18d3f722c124c513ffb5e5eaff0cc4f6d3aa4bda38e691b8600c9f"}, - {file = "ruff-0.3.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:122de171a147c76ada00f76df533b54676f6e321e61bd8656ae54be326c10296"}, - {file = "ruff-0.3.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d80a6b18a6c3b6ed25b71b05eba183f37d9bc8b16ace9e3d700997f00b74660b"}, - {file = "ruff-0.3.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a7b6e63194c68bca8e71f81de30cfa6f58ff70393cf45aab4c20f158227d5936"}, - {file = "ruff-0.3.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a759d33a20c72f2dfa54dae6e85e1225b8e302e8ac655773aff22e542a300985"}, - {file = "ruff-0.3.5-py3-none-win32.whl", hash = "sha256:9d8605aa990045517c911726d21293ef4baa64f87265896e491a05461cae078d"}, - {file = "ruff-0.3.5-py3-none-win_amd64.whl", hash = "sha256:dc56bb16a63c1303bd47563c60482a1512721053d93231cf7e9e1c6954395a0e"}, - {file = "ruff-0.3.5-py3-none-win_arm64.whl", hash = "sha256:faeeae9905446b975dcf6d4499dc93439b131f1443ee264055c5716dd947af55"}, - {file = "ruff-0.3.5.tar.gz", hash = "sha256:a067daaeb1dc2baf9b82a32dae67d154d95212080c80435eb052d95da647763d"}, + {file = "ruff-0.4.2-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:8d14dc8953f8af7e003a485ef560bbefa5f8cc1ad994eebb5b12136049bbccc5"}, + {file = "ruff-0.4.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:24016ed18db3dc9786af103ff49c03bdf408ea253f3cb9e3638f39ac9cf2d483"}, + {file = "ruff-0.4.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e2e06459042ac841ed510196c350ba35a9b24a643e23db60d79b2db92af0c2b"}, + {file = "ruff-0.4.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3afabaf7ba8e9c485a14ad8f4122feff6b2b93cc53cd4dad2fd24ae35112d5c5"}, + {file = "ruff-0.4.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:799eb468ea6bc54b95527143a4ceaf970d5aa3613050c6cff54c85fda3fde480"}, + {file = "ruff-0.4.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:ec4ba9436a51527fb6931a8839af4c36a5481f8c19e8f5e42c2f7ad3a49f5069"}, + {file = "ruff-0.4.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6a2243f8f434e487c2a010c7252150b1fdf019035130f41b77626f5655c9ca22"}, + {file = "ruff-0.4.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8772130a063f3eebdf7095da00c0b9898bd1774c43b336272c3e98667d4fb8fa"}, + {file = "ruff-0.4.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ab165ef5d72392b4ebb85a8b0fbd321f69832a632e07a74794c0e598e7a8376"}, + {file = "ruff-0.4.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1f32cadf44c2020e75e0c56c3408ed1d32c024766bd41aedef92aa3ca28eef68"}, + {file = "ruff-0.4.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:22e306bf15e09af45ca812bc42fa59b628646fa7c26072555f278994890bc7ac"}, + {file = "ruff-0.4.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:82986bb77ad83a1719c90b9528a9dd663c9206f7c0ab69282af8223566a0c34e"}, + {file = "ruff-0.4.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:652e4ba553e421a6dc2a6d4868bc3b3881311702633eb3672f9f244ded8908cd"}, + {file = "ruff-0.4.2-py3-none-win32.whl", hash = "sha256:7891ee376770ac094da3ad40c116258a381b86c7352552788377c6eb16d784fe"}, + {file = "ruff-0.4.2-py3-none-win_amd64.whl", hash = "sha256:5ec481661fb2fd88a5d6cf1f83403d388ec90f9daaa36e40e2c003de66751798"}, + {file = "ruff-0.4.2-py3-none-win_arm64.whl", hash = "sha256:cbd1e87c71bca14792948c4ccb51ee61c3296e164019d2d484f3eaa2d360dfaf"}, + {file = "ruff-0.4.2.tar.gz", hash = "sha256:33bcc160aee2520664bc0859cfeaebc84bb7323becff3f303b8f1f2d81cb4edc"}, ] [[package]] name = "setuptools" -version = "69.2.0" +version = "69.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, - {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, + {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, + {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -1830,28 +1830,28 @@ files = [ [[package]] name = "traitlets" -version = "5.14.2" +version = "5.14.3" description = "Traitlets Python configuration system" optional = false python-versions = ">=3.8" files = [ - {file = "traitlets-5.14.2-py3-none-any.whl", hash = "sha256:fcdf85684a772ddeba87db2f398ce00b40ff550d1528c03c14dbf6a02003cd80"}, - {file = "traitlets-5.14.2.tar.gz", hash = "sha256:8cdd83c040dab7d1dee822678e5f5d100b514f7b72b01615b26fc5718916fdf9"}, + {file = "traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"}, + {file = "traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7"}, ] [package.extras] docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] -test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<8.1)", "pytest-mock", "pytest-mypy-testing"] +test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<8.2)", "pytest-mock", "pytest-mypy-testing"] [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.11.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, ] [[package]] @@ -1899,17 +1899,17 @@ files = [ [[package]] name = "websocket-client" -version = "1.7.0" +version = "1.8.0" description = "WebSocket client for Python with low level API options" optional = false python-versions = ">=3.8" files = [ - {file = "websocket-client-1.7.0.tar.gz", hash = "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6"}, - {file = "websocket_client-1.7.0-py3-none-any.whl", hash = "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"}, + {file = "websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526"}, + {file = "websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da"}, ] [package.extras] -docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] +docs = ["Sphinx (>=6.0)", "myst-parser (>=2.0.0)", "sphinx-rtd-theme (>=1.1.0)"] optional = ["python-socks", "wsaccel"] test = ["websockets"] @@ -2012,4 +2012,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<4.0" -content-hash = "d92ca13cc9f26032bcc2fe7796a4bae9ff84aa82886773e307158fde6e14891f" +content-hash = "1f091d9340b0d96c5645eded33fa2e5b4edfe796def64b05a3becc41aa6a091d" diff --git a/pyproject.toml b/pyproject.toml index e0ee24d9..c664ed7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ optional = true [tool.poetry.group.fmt.dependencies] black = "^22.3.0" -ruff = ">=0.0.157" +ruff = ">=0.1.0" pyright = "^1.1.300" [tool.poetry.group.lint] @@ -57,7 +57,7 @@ optional = true [tool.poetry.group.lint.dependencies] black = "^22.3.0" -ruff = ">=0.0.157" +ruff = ">=0.1.0" codespell = ">=2.2.2" pyright = "^1.1.301" @@ -93,8 +93,8 @@ pyright = "^1.1.301" [tool.ruff] line-length = 99 -select = ["E", "W", "F", "C", "N", "D", "I001"] -extend-ignore = [ +lint.select = ["E", "W", "F", "C", "N", "D", "I001"] +lint.extend-ignore = [ "D203", "D204", "D213", @@ -108,13 +108,13 @@ extend-ignore = [ "D409", "D413", ] -ignore = ["E501", "D107"] +lint.ignore = ["E501", "D107"] extend-exclude = ["__pycache__", "*.egg_info", "tests/integration/app-charm/lib"] -per-file-ignores = {"tests/*" = ["D100","D101","D102","D103","D104", "E999"], "src/literals.py" = ["D101"]} +lint.per-file-ignores = {"tests/*" = ["D100","D101","D102","D103","D104", "E999"], "src/literals.py" = ["D101"]} target-version="py310" src = ["src", "tests"] -[tool.ruff.mccabe] +[tool.ruff.lint.mccabe] max-complexity = 10 [tool.pyright] diff --git a/requirements.txt b/requirements.txt index 8b4ccb66..eb65604f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,20 @@ attrs==23.2.0 ; python_version >= "3.8" and python_version < "4.0" cffi==1.16.0 ; python_version >= "3.8" and python_version < "4.0" and platform_python_implementation != "PyPy" -cosl==0.0.10 ; python_version >= "3.8" and python_version < "4.0" +cosl==0.0.11 ; python_version >= "3.8" and python_version < "4.0" cryptography==42.0.5 ; python_version >= "3.8" and python_version < "4.0" importlib-resources==6.4.0 ; python_version >= "3.8" and python_version < "3.9" jsonschema-specifications==2023.12.1 ; python_version >= "3.8" and python_version < "4.0" -jsonschema==4.21.1 ; python_version >= "3.8" and python_version < "4.0" +jsonschema==4.22.0 ; python_version >= "3.8" and python_version < "4.0" kazoo==2.10.0 ; python_version >= "3.8" and python_version < "4.0" -ops==2.12.0 ; python_version >= "3.8" and python_version < "4.0" +ops==2.13.0 ; python_version >= "3.8" and python_version < "4.0" pkgutil-resolve-name==1.3.10 ; python_version >= "3.8" and python_version < "3.9" pure-sasl==0.6.2 ; python_version >= "3.8" and python_version < "4.0" pycparser==2.22 ; python_version >= "3.8" and python_version < "4.0" and platform_python_implementation != "PyPy" -pydantic==1.10.14 ; python_version >= "3.8" and python_version < "4.0" +pydantic==1.10.15 ; python_version >= "3.8" and python_version < "4.0" pyyaml==6.0.1 ; python_version >= "3.8" and python_version < "4.0" -referencing==0.34.0 ; python_version >= "3.8" and python_version < "4.0" +referencing==0.35.0 ; python_version >= "3.8" and python_version < "4.0" rpds-py==0.18.0 ; python_version >= "3.8" and python_version < "4.0" tenacity==8.2.3 ; python_version >= "3.8" and python_version < "4.0" -typing-extensions==4.10.0 ; python_version >= "3.8" and python_version < "4.0" -websocket-client==1.7.0 ; python_version >= "3.8" and python_version < "4.0" +typing-extensions==4.11.0 ; python_version >= "3.8" and python_version < "4.0" +websocket-client==1.8.0 ; python_version >= "3.8" and python_version < "4.0" zipp==3.18.1 ; python_version >= "3.8" and python_version < "3.9" diff --git a/src/charm.py b/src/charm.py index f4164004..aa82ac23 100755 --- a/src/charm.py +++ b/src/charm.py @@ -12,12 +12,13 @@ from charms.operator_libs_linux.v0 import sysctl from charms.operator_libs_linux.v1.snap import SnapError from charms.rolling_ops.v0.rollingops import RollingOpsManager, RunWithLock -from ops.charm import StorageAttachedEvent, StorageDetachingEvent, StorageEvent +from ops.charm import SecretChangedEvent, StorageAttachedEvent, StorageDetachingEvent, StorageEvent from ops.framework import EventBase from ops.main import main from ops.model import ActiveStatus, StatusBase from core.cluster import ClusterState +from core.models import Substrates from core.structured_config import CharmConfig from events.password_actions import PasswordActionEvents from events.provider import KafkaProvider @@ -35,10 +36,10 @@ OS_REQUIREMENTS, PEER, REL_NAME, + SUBSTRATE, USER, DebugLevel, Status, - Substrate, ) from managers.auth import AuthManager from managers.config import KafkaConfigManager @@ -56,7 +57,7 @@ class KafkaCharm(TypedCharmBase[CharmConfig]): def __init__(self, *args): super().__init__(*args) self.name = CHARM_KEY - self.substrate: Substrate = "vm" + self.substrate: Substrates = SUBSTRATE self.workload = KafkaWorkload() self.state = ClusterState(self, substrate=self.substrate) @@ -111,6 +112,7 @@ def __init__(self, *args): self.framework.observe(getattr(self.on, "config_changed"), self._on_config_changed) self.framework.observe(getattr(self.on, "update_status"), self._on_update_status) self.framework.observe(getattr(self.on, "remove"), self._on_remove) + self.framework.observe(getattr(self.on, "secret_changed"), self._on_secret_changed) self.framework.observe(self.on[PEER].relation_changed, self._on_config_changed) @@ -207,7 +209,6 @@ def _on_config_changed(self, event: EventBase) -> None: callback_override="_disable_enable_restart" ) else: - logger.info("Acquiring lock from _on_config_changed...") self.on[f"{self.restart.name}"].acquire_lock.emit() # update client_properties whenever possible @@ -215,9 +216,9 @@ def _on_config_changed(self, event: EventBase) -> None: # If Kafka is related to client charms, update their information. if self.model.relations.get(REL_NAME, None) and self.unit.is_leader(): - self.provider.update_connection_info() + self.update_client_data() - def _on_update_status(self, event: EventBase) -> None: + def _on_update_status(self, _: EventBase) -> None: """Handler for `update-status` events.""" if not self.healthy or not self.upgrade.idle: return @@ -228,7 +229,7 @@ def _on_update_status(self, event: EventBase) -> None: # NOTE for situations like IP change and late integration with rack-awareness charm. # If properties have changed, the broker will restart. - self._on_config_changed(event) + self.on.config_changed.emit() try: if not self.health.machine_configured(): @@ -245,6 +246,18 @@ def _on_remove(self, _) -> None: """Handler for stop.""" self.sysctl_config.remove() + def _on_secret_changed(self, event: SecretChangedEvent) -> None: + """Handler for `secret_changed` events.""" + if not event.secret.label or not self.state.cluster.relation: + return + + if event.secret.label == self.state.cluster.data_interface._generate_secret_label( + PEER, + self.state.cluster.relation.id, + "extra", # pyright: ignore[reportArgumentType] -- Changes with the https://github.com/canonical/data-platform-libs/issues/124 + ): + self.on.config_changed.emit() + def _on_storage_attached(self, event: StorageAttachedEvent) -> None: """Handler for `storage_attached` events.""" # new dirs won't be used until topic partitions are assigned to it @@ -256,7 +269,7 @@ def _on_storage_attached(self, event: StorageAttachedEvent) -> None: self.workload.exec(f"chmod -R 770 {self.workload.paths.data_path}") self._on_config_changed(event) - def _on_storage_detaching(self, event: StorageDetachingEvent) -> None: + def _on_storage_detaching(self, _: StorageDetachingEvent) -> None: """Handler for `storage_detaching` events.""" # in the case where there may be replication recovery may be possible if self.state.brokers and len(self.state.brokers) > 1: @@ -264,7 +277,7 @@ def _on_storage_detaching(self, event: StorageDetachingEvent) -> None: else: self._set_status(Status.REMOVED_STORAGE_NO_REPL) - self._on_config_changed(event) + self.on.config_changed.emit() def _restart(self, event: EventBase) -> None: """Handler for `rolling_ops` restart events.""" @@ -327,6 +340,31 @@ def healthy(self) -> bool: return True + def update_client_data(self) -> None: + """Writes necessary relation data to all related client applications.""" + if not self.unit.is_leader() or not self.healthy: + return + + for client in self.state.clients: + if not client.password: + logger.debug( + f"Skipping update of {client.app.name}, user has not yet been added..." + ) + continue + + client.update( + { + "endpoints": client.bootstrap_server, + "zookeeper-uris": client.zookeeper_uris, + "consumer-group-prefix": client.consumer_group_prefix, + "topic": client.topic, + "username": client.username, + "password": client.password, + "tls": client.tls, + "tls-ca": client.tls, # TODO: fix tls-ca + } + ) + def _set_status(self, key: Status) -> None: """Sets charm status.""" status: StatusBase = key.value.status diff --git a/src/core/cluster.py b/src/core/cluster.py index 5a957eda..a25e2134 100644 --- a/src/core/cluster.py +++ b/src/core/cluster.py @@ -5,27 +5,46 @@ """Objects representing the state of KafkaCharm.""" import os - +from functools import cached_property + +from charms.data_platform_libs.v0.data_interfaces import ( + DatabaseRequirerData, + DataPeerData, + DataPeerOtherUnitData, + DataPeerUnitData, + KafkaProviderData, +) from ops import Framework, Object, Relation +from ops.model import Unit -from core.models import KafkaBroker, KafkaCluster, ZooKeeper +from core.models import KafkaBroker, KafkaClient, KafkaCluster, ZooKeeper from literals import ( INTERNAL_USERS, PEER, REL_NAME, + SECRETS_UNIT, SECURITY_PROTOCOL_PORTS, ZK, Status, - Substrate, + Substrates, ) class ClusterState(Object): - """Properties and relations of the charm.""" + """Collection of global cluster state for the Kafka services.""" - def __init__(self, charm: Framework | Object, substrate: Substrate): + def __init__(self, charm: Framework | Object, substrate: Substrates): super().__init__(parent=charm, key="charm_state") - self.substrate: Substrate = substrate + self.substrate: Substrates = substrate + + self.peer_app_interface = DataPeerData(self.model, relation_name=PEER) + self.peer_unit_interface = DataPeerUnitData( + self.model, relation_name=PEER, additional_secret_fields=SECRETS_UNIT + ) + self.zookeeper_requires_interface = DatabaseRequirerData( + self.model, relation_name=ZK, database_name=f"/{self.model.app.name}" + ) + self.client_provider_interface = KafkaProviderData(self.model, relation_name=REL_NAME) # --- RELATIONS --- @@ -47,17 +66,34 @@ def client_relations(self) -> set[Relation]: # --- CORE COMPONENTS --- @property - def broker(self) -> KafkaBroker: - """The server state of the current running Unit.""" + def unit_broker(self) -> KafkaBroker: + """The broker state of the current running Unit.""" return KafkaBroker( - relation=self.peer_relation, component=self.model.unit, substrate=self.substrate + relation=self.peer_relation, + data_interface=self.peer_unit_interface, + component=self.model.unit, + substrate=self.substrate, ) + @cached_property + def peer_units_data_interfaces(self) -> dict[Unit, DataPeerOtherUnitData]: + """The cluster peer relation.""" + if not self.peer_relation or not self.peer_relation.units: + return {} + + return { + unit: DataPeerOtherUnitData(model=self.model, unit=unit, relation_name=PEER) + for unit in self.peer_relation.units + } + @property def cluster(self) -> KafkaCluster: """The cluster state of the current running App.""" return KafkaCluster( - relation=self.peer_relation, component=self.model.app, substrate=self.substrate + relation=self.peer_relation, + data_interface=self.peer_app_interface, + component=self.model.app, + substrate=self.substrate, ) @property @@ -67,29 +103,54 @@ def brokers(self) -> set[KafkaBroker]: Returns: Set of KafkaBrokers in the current peer relation, including the running unit server. """ - if not self.peer_relation: - return set() - - servers = set() - for unit in self.peer_relation.units: - servers.add( - KafkaBroker(relation=self.peer_relation, component=unit, substrate=self.substrate) + brokers = set() + for unit, data_interface in self.peer_units_data_interfaces.items(): + brokers.add( + KafkaBroker( + relation=self.peer_relation, + data_interface=data_interface, + component=unit, + substrate=self.substrate, + ) ) - servers.add(self.broker) + brokers.add(self.unit_broker) - return servers + return brokers @property def zookeeper(self) -> ZooKeeper: """The ZooKeeper relation state.""" return ZooKeeper( relation=self.zookeeper_relation, - component=self.model.app, + data_interface=self.zookeeper_requires_interface, substrate=self.substrate, - local_app=self.model.app, - local_unit=self.model.unit, + local_app=self.cluster.app, ) + @property + def clients(self) -> set[KafkaClient]: + """The state for all related client Applications.""" + clients = set() + for relation in self.client_relations: + if not relation.app: + continue + + clients.add( + KafkaClient( + relation=relation, + data_interface=self.client_provider_interface, + component=relation.app, + substrate=self.substrate, + local_app=self.cluster.app, + bootstrap_server=self.bootstrap_server, + password=self.cluster.client_passwords.get(f"relation-{relation.id}", ""), + tls="enabled" if self.cluster.tls_enabled else "disabled", + zookeeper_uris=self.zookeeper.uris, + ) + ) + + return clients + # ---- GENERAL VALUES ---- @property @@ -121,21 +182,21 @@ def port(self) -> int: """Return the port to be used internally.""" return ( SECURITY_PROTOCOL_PORTS["SASL_SSL"].client - if (self.cluster.tls_enabled and self.broker.certificate) + if (self.cluster.tls_enabled and self.unit_broker.certificate) else SECURITY_PROTOCOL_PORTS["SASL_PLAINTEXT"].client ) @property - def bootstrap_server(self) -> list[str]: + def bootstrap_server(self) -> str: """The current Kafka uris formatted for the `bootstrap-server` command flag. Returns: List of `bootstrap-server` servers """ if not self.peer_relation: - return [] + return "" - return sorted([f"{host}:{self.port}" for host in self.unit_hosts]) + return ",".join(sorted([f"{broker.host}:{self.port}" for broker in self.brokers])) @property def log_dirs(self) -> str: @@ -146,12 +207,6 @@ def log_dirs(self) -> str: """ return ",".join([os.fspath(storage.location) for storage in self.model.storages["data"]]) - @property - def unit_hosts(self) -> list[str]: - """Return list of application unit hosts.""" - hosts = [broker.host for broker in self.brokers] - return hosts - @property def planned_units(self) -> int: """Return the planned units for the charm.""" @@ -167,7 +222,7 @@ def ready_to_start(self) -> Status: if not self.peer_relation: return Status.NO_PEER_RELATION - if not self.zookeeper.zookeeper_related: + if not self.zookeeper: return Status.ZK_NOT_RELATED if not self.zookeeper.zookeeper_connected: @@ -177,6 +232,9 @@ def ready_to_start(self) -> Status: if self.cluster.tls_enabled ^ self.zookeeper.tls: return Status.ZK_TLS_MISMATCH + if self.cluster.tls_enabled and not self.unit_broker.certificate: + return Status.NO_CERT + if not self.cluster.internal_user_credentials: return Status.NO_BROKER_CREDS diff --git a/src/core/models.py b/src/core/models.py index cf435f4d..567eadcd 100644 --- a/src/core/models.py +++ b/src/core/models.py @@ -5,51 +5,90 @@ """Collection of state objects for the Kafka relations, apps and units.""" import logging -from typing import MutableMapping +from charms.data_platform_libs.v0.data_interfaces import Data, DataPeerData, DataPeerUnitData from charms.zookeeper.v0.client import QuorumLeaderNotFoundError, ZooKeeperManager -from kazoo.exceptions import AuthFailedError, NoNodeError +from kazoo.client import AuthFailedError, NoNodeError from ops.model import Application, Relation, Unit -from tenacity import retry, retry_if_not_result, stop_after_attempt, wait_fixed +from tenacity import retry +from tenacity.retry import retry_if_not_result +from tenacity.stop import stop_after_attempt +from tenacity.wait import wait_fixed +from typing_extensions import override -from literals import INTERNAL_USERS, Substrate +from literals import INTERNAL_USERS, SECRETS_APP, Substrates logger = logging.getLogger(__name__) -class StateBase: - """Base state object.""" +class RelationState: + """Relation state object.""" def __init__( - self, relation: Relation | None, component: Unit | Application, substrate: Substrate + self, + relation: Relation | None, + data_interface: Data, + component: Unit | Application | None, + substrate: Substrates, ): self.relation = relation - self.component = component + self.data_interface = data_interface + self.component = ( + component # FIXME: remove, and use _fetch_my_relation_data defaults wheren needed + ) self.substrate = substrate + self.relation_data = ( + self.data_interface.as_dict(self.relation.id) if self.relation else {} + ) # FIXME: mappingproxytype? - @property - def relation_data(self) -> MutableMapping[str, str]: - """The raw relation data.""" - if not self.relation: - return {} - - return self.relation.data[self.component] + def __bool__(self) -> bool: + """Boolean evaluation based on the existence of self.relation.""" + try: + return bool(self.relation) + except AttributeError: + return False def update(self, items: dict[str, str]) -> None: """Writes to relation_data.""" - if not self.relation: - return + delete_fields = [key for key in items if not items[key]] + update_content = {k: items[k] for k in items if k not in delete_fields} + + self.relation_data.update(update_content) - self.relation_data.update(items) + for field in delete_fields: + del self.relation_data[field] -class KafkaCluster(StateBase): +class KafkaCluster(RelationState): """State collection metadata for the peer relation.""" - def __init__(self, relation: Relation | None, component: Application, substrate: Substrate): - super().__init__(relation, component, substrate) + def __init__( + self, + relation: Relation | None, + data_interface: DataPeerData, + component: Application, + substrate: Substrates, + ): + super().__init__(relation, data_interface, component, substrate) + self.data_interface = data_interface self.app = component + @override + def update(self, items: dict[str, str]) -> None: + """Overridden update to allow for same interface, but writing to local app bag.""" + if not self.relation: + return + + for key, value in items.items(): + # note: relation- check accounts for dynamically created secrets + if key in SECRETS_APP or key.startswith("relation-"): + if value: + self.data_interface.set_secret(self.relation.id, key, value) + else: + self.data_interface.delete_secret(self.relation.id, key) + else: + self.data_interface.update_relation_data(self.relation.id, {key: value}) + @property def internal_user_credentials(self) -> dict[str, str]: """The charm internal usernames and passwords, e.g `sync` and `admin`. @@ -94,11 +133,18 @@ def mtls_enabled(self) -> bool: return self.relation_data.get("mtls", "disabled") == "enabled" -class KafkaBroker(StateBase): - """State collection metadata for a charm unit.""" +class KafkaBroker(RelationState): + """State collection metadata for a unit.""" - def __init__(self, relation: Relation | None, component: Unit, substrate: Substrate): - super().__init__(relation, component, substrate) + def __init__( + self, + relation: Relation | None, + data_interface: DataPeerUnitData, + component: Unit, + substrate: Substrates, + ): + super().__init__(relation, data_interface, component, substrate) + self.data_interface = data_interface self.unit = component @property @@ -107,7 +153,7 @@ def unit_id(self) -> int: e.g kafka/2 --> 2 """ - return int(self.component.name.split("/")[1]) + return int(self.unit.name.split("/")[1]) @property def host(self) -> str: @@ -118,146 +164,158 @@ def host(self) -> str: if host := self.relation_data.get(key, ""): break if self.substrate == "k8s": - host = f"{self.component.name.split('/')[0]}-{self.unit_id}.{self.component.name.split('/')[0]}-endpoints" + host = f"{self.unit.name.split('/')[0]}-{self.unit_id}.{self.unit.name.split('/')[0]}-endpoints" return host # --- TLS --- @property - def private_key(self) -> str | None: + def private_key(self) -> str: """The unit private-key set during `certificates_joined`. Returns: String of key contents None if key not yet generated """ - return self.relation_data.get("private-key") + return self.relation_data.get("private-key", "") @property - def csr(self) -> str | None: + def csr(self) -> str: """The unit cert signing request. Returns: String of csr contents None if csr not yet generated """ - return self.relation_data.get("csr") + return self.relation_data.get("csr", "") @property - def certificate(self) -> str | None: + def certificate(self) -> str: """The signed unit certificate from the provider relation. Returns: String of cert contents in PEM format None if cert not yet generated/signed """ - return self.relation_data.get("certificate") + return self.relation_data.get("certificate", "") @property - def ca(self) -> str | None: + def ca(self) -> str: """The ca used to sign unit cert. Returns: String of ca contents in PEM format None if cert not yet generated/signed """ - return self.relation_data.get("ca") + # defaults to ca for backwards compatibility after field change introduced with secrets + return self.relation_data.get("ca-cert") or self.relation_data.get("ca", "") @property - def keystore_password(self) -> str | None: + def keystore_password(self) -> str: """The unit keystore password set during `certificates_joined`. Returns: String of password None if password not yet generated """ - return self.relation_data.get("keystore-password") + return self.relation_data.get("keystore-password", "") @property - def truststore_password(self) -> str | None: + def truststore_password(self) -> str: """The unit truststore password set during `certificates_joined`. Returns: String of password None if password not yet generated """ - return self.relation_data.get("truststore-password") + return self.relation_data.get("truststore-password", "") -class ZooKeeper(StateBase): +class ZooKeeper(RelationState): """State collection metadata for a the Zookeeper relation.""" def __init__( self, relation: Relation | None, - component: Application, - substrate: Substrate, - local_unit: Unit, + data_interface: Data, + substrate: Substrates, local_app: Application | None = None, ): - super().__init__(relation, component, substrate) + super().__init__(relation, data_interface, None, substrate) self._local_app = local_app - self._local_unit = local_unit - - # APPLICATION DATA - - @property - def remote_app_data(self) -> MutableMapping[str, str]: - """Zookeeper relation data object.""" - if not self.relation or not self.relation.app: - return {} - - return self.relation.data[self.relation.app] - - @property - def app_data(self) -> MutableMapping[str, str]: - """Zookeeper relation data object.""" - if not self.relation or not self._local_app: - return {} - - return self.relation.data[self._local_app] - - # --- RELATION PROPERTIES --- - - @property - def zookeeper_related(self) -> bool: - """Checks if there is a relation with ZooKeeper. - - Returns: - True if there is a ZooKeeper relation. Otherwise False - """ - return bool(self.relation) @property def username(self) -> str: """Username to connect to ZooKeeper.""" - return self.remote_app_data.get("username", "") + if not self.relation: + return "" + + return ( + self.data_interface.fetch_relation_field( + relation_id=self.relation.id, field="username" + ) + or "" + ) @property def password(self) -> str: """Password of the ZooKeeper user.""" - return self.remote_app_data.get("password", "") + if not self.relation: + return "" + + return ( + self.data_interface.fetch_relation_field( + relation_id=self.relation.id, field="password" + ) + or "" + ) @property def endpoints(self) -> str: """IP/host where ZooKeeper is located.""" - return self.remote_app_data.get("endpoints", "") + if not self.relation: + return "" + + return ( + self.data_interface.fetch_relation_field( + relation_id=self.relation.id, field="endpoints" + ) + or "" + ) @property def chroot(self) -> str: """Path allocated for Kafka on ZooKeeper.""" - return self.remote_app_data.get("chroot", "") + if not self.relation: + return "" + + return ( + self.data_interface.fetch_relation_field(relation_id=self.relation.id, field="chroot") + or "" + ) @property def uris(self) -> str: """Comma separated connection string, containing endpoints + chroot.""" - return self.remote_app_data.get("uris", "") + if not self.relation: + return "" + + return ( + self.data_interface.fetch_relation_field(relation_id=self.relation.id, field="uris") + or "" + ) @property def tls(self) -> bool: """Check if TLS is enabled on ZooKeeper.""" - return bool(self.remote_app_data.get("tls", "disabled") == "enabled") + if not self.relation: + return False + + return ( + self.data_interface.fetch_relation_field(relation_id=self.relation.id, field="tls") + or "" + ) == "enabled" @property def connect(self) -> str: @@ -296,21 +354,90 @@ def zookeeper_version(self) -> str: ) def broker_active(self) -> bool: """Checks if broker id is recognised as active by ZooKeeper.""" - broker_id = self._local_unit.name.split("/")[1] - brokers = self.get_active_brokers() - return f"{self.chroot}/brokers/ids/{broker_id}" in brokers - - def get_active_brokers(self) -> set[str]: - """Gets all brokers currently connected to ZooKeeper.""" + broker_id = self.data_interface.local_unit.name.split("/")[1] hosts = self.endpoints.split(",") - zk = ZooKeeperManager(hosts=hosts, username=self.username, password=self.password) path = f"{self.chroot}/brokers/ids/" + zk = ZooKeeperManager(hosts=hosts, username=self.username, password=self.password) try: brokers = zk.leader_znodes(path=path) - # auth might not be ready with ZK after relation yet except (NoNodeError, AuthFailedError, QuorumLeaderNotFoundError) as e: logger.debug(str(e)) - return set() + brokers = set() + + return f"{path}{broker_id}" in brokers + + +class KafkaClient(RelationState): + """State collection metadata for a single related client application.""" + + def __init__( + self, + relation: Relation | None, + data_interface: Data, + component: Application, + substrate: Substrates, + local_app: Application | None = None, + bootstrap_server: str = "", + password: str = "", # nosec: B107 + tls: str = "", + zookeeper_uris: str = "", + ): + super().__init__(relation, data_interface, component, substrate) + self.app = component + self._local_app = local_app + self._bootstrap_server = bootstrap_server + self._password = password + self._tls = tls + self._zookeeper_uris = zookeeper_uris - return brokers + @property + def username(self) -> str: + """The generated username for the client application.""" + return f"relation-{getattr(self.relation, 'id', '')}" + + @property + def bootstrap_server(self) -> str: + """The Kafka server endpoints for the client application to connect with.""" + return self._bootstrap_server + + @property + def password(self) -> str: + """The generated password for the client application.""" + return self._password + + @property + def consumer_group_prefix(self) -> str: + """The assigned consumer group prefix for a client application presenting consumer role.""" + return self.relation_data.get( + "consumer-group-prefix", + f"{self.username}-" if "consumer" in self.extra_user_roles else "", + ) + + @property + def tls(self) -> str: + """Flag to confirm whether or not ZooKeeper has TLS enabled. + + Returns: + String of either 'enabled' or 'disabled' + """ + return self._tls + + @property + def zookeeper_uris(self) -> str: + """The ZooKeeper connection endpoints for the client application to connect with.""" + return self._zookeeper_uris + + @property + def topic(self) -> str: + """The ZooKeeper connection endpoints for the client application to connect with.""" + return self.relation_data.get("topic", "") + + @property + def extra_user_roles(self) -> str: + """The client defined roles for their application. + + Can be any comma-delimited selection of `producer`, `consumer` and `admin`. + When `admin` is set, the Kafka charm interprets this as a new super.user. + """ + return self.relation_data.get("extra-user-roles", "") diff --git a/src/events/provider.py b/src/events/provider.py index 3c57519d..93c1ab2a 100644 --- a/src/events/provider.py +++ b/src/events/provider.py @@ -5,10 +5,13 @@ """KafkaProvider class and methods.""" import logging -import subprocess +import subprocess # nosec B404 from typing import TYPE_CHECKING -from charms.data_platform_libs.v0.data_interfaces import KafkaProvides, TopicRequestedEvent +from charms.data_platform_libs.v0.data_interfaces import ( + KafkaProviderEventHandlers, + TopicRequestedEvent, +) from ops.charm import RelationBrokenEvent, RelationCreatedEvent from ops.framework import Object from ops.pebble import ExecError @@ -27,7 +30,9 @@ class KafkaProvider(Object): def __init__(self, charm) -> None: super().__init__(charm, "kafka_client") self.charm: "KafkaCharm" = charm - self.kafka_provider = KafkaProvides(self.charm, REL_NAME) + self.kafka_provider = KafkaProviderEventHandlers( + self.charm, self.charm.state.client_provider_interface + ) self.framework.observe(self.charm.on[REL_NAME].relation_created, self._on_relation_created) self.framework.observe(self.charm.on[REL_NAME].relation_broken, self._on_relation_broken) @@ -48,52 +53,43 @@ def on_topic_requested(self, event: TopicRequestedEvent): if not self.charm.unit.is_leader() or not self.charm.state.peer_relation: return - extra_user_roles = event.extra_user_roles or "" - topic = event.topic or "" - relation = event.relation - username = f"relation-{relation.id}" - password = ( - self.charm.state.cluster.client_passwords.get(username) - or self.charm.workload.generate_password() - ) - bootstrap_server = self.charm.state.bootstrap_server - zookeeper_uris = self.charm.state.zookeeper.connect - tls = "enabled" if self.charm.state.cluster.tls_enabled else "disabled" + requesting_client = None + for client in self.charm.state.clients: + if event.relation == client.relation: + requesting_client = client + break - consumer_group_prefix = ( - event.consumer_group_prefix or f"{username}-" if "consumer" in extra_user_roles else "" - ) + if not requesting_client: + event.defer() + return + + password = client.password or self.charm.workload.generate_password() # catching error here in case listeners not established for bootstrap-server auth try: self.charm.auth_manager.add_user( - username=username, + username=client.username, password=password, ) except (subprocess.CalledProcessError, ExecError): - logger.warning(f"unable to create user {username} just yet") + logger.warning(f"unable to create user {client.username} just yet") event.defer() return # non-leader units need cluster_config_changed event to update their super.users - self.charm.state.cluster.update({username: password}) + self.charm.state.cluster.update({client.username: password}) self.charm.auth_manager.update_user_acls( - username=username, - topic=topic, - extra_user_roles=extra_user_roles, - group=consumer_group_prefix, + username=client.username, + topic=client.topic, + extra_user_roles=client.extra_user_roles, + group=client.consumer_group_prefix, ) # non-leader units need cluster_config_changed event to update their super.users self.charm.state.cluster.update({"super-users": self.charm.state.super_users}) - self.kafka_provider.set_bootstrap_server(relation.id, ",".join(bootstrap_server)) - self.kafka_provider.set_consumer_group_prefix(relation.id, consumer_group_prefix) - self.kafka_provider.set_credentials(relation.id, username, password) - self.kafka_provider.set_tls(relation.id, tls) - self.kafka_provider.set_zookeeper_uris(relation.id, zookeeper_uris) - self.kafka_provider.set_topic(relation.id, topic) + self.charm.update_client_data() def _on_relation_created(self, event: RelationCreatedEvent) -> None: """Handler for `kafka-client-relation-created` event.""" @@ -107,11 +103,12 @@ def _on_relation_broken(self, event: RelationBrokenEvent) -> None: Args: event: the event from a related client application needing a user """ - # don't remove anything if app is going down - if self.charm.app.planned_units == 0: - return - - if not self.charm.unit.is_leader() or not self.charm.state.peer_relation: + if ( + # don't remove anything if app is going down + self.charm.app.planned_units == 0 + or not self.charm.unit.is_leader() + or not self.charm.state.cluster + ): return if not self.charm.healthy: @@ -120,27 +117,12 @@ def _on_relation_broken(self, event: RelationBrokenEvent) -> None: if event.relation.app != self.charm.app or not self.charm.app.planned_units() == 0: username = f"relation-{event.relation.id}" + self.charm.auth_manager.remove_all_user_acls(username=username) self.charm.auth_manager.delete_user(username=username) + # non-leader units need cluster_config_changed event to update their super.users # update on the peer relation data will trigger an update of server properties on all units self.charm.state.cluster.update({username: ""}) - def update_connection_info(self): - """Updates all relations with current endpoints, bootstrap-server and tls data. - - If information didn't change, no events will trigger. - """ - bootstrap_server = self.charm.state.bootstrap_server - zookeeper_uris = self.charm.state.zookeeper.connect - tls = "enabled" if self.charm.state.cluster.tls_enabled else "disabled" - - for relation in self.charm.model.relations[REL_NAME]: - if f"relation-{relation.id}" in self.charm.state.cluster.client_passwords: - self.kafka_provider.set_bootstrap_server( - relation_id=relation.id, bootstrap_server=",".join(bootstrap_server) - ) - self.kafka_provider.set_tls(relation_id=relation.id, tls=tls) - self.kafka_provider.set_zookeeper_uris( - relation_id=relation.id, zookeeper_uris=zookeeper_uris - ) + self.charm.update_client_data() diff --git a/src/events/tls.py b/src/events/tls.py index 4bdd8d38..41daef77 100644 --- a/src/events/tls.py +++ b/src/events/tls.py @@ -94,16 +94,18 @@ def _tls_relation_created(self, _) -> None: def _tls_relation_joined(self, _) -> None: """Handler for `certificates_relation_joined` event.""" # generate unit private key if not already created by action - if not self.charm.state.broker.private_key: - self.charm.state.broker.update({"private-key": generate_private_key().decode("utf-8")}) + if not self.charm.state.unit_broker.private_key: + self.charm.state.unit_broker.update( + {"private-key": generate_private_key().decode("utf-8")} + ) # generate unit private key if not already created by action - if not self.charm.state.broker.keystore_password: - self.charm.state.broker.update( + if not self.charm.state.unit_broker.keystore_password: + self.charm.state.unit_broker.update( {"keystore-password": self.charm.workload.generate_password()} ) - if not self.charm.state.broker.truststore_password: - self.charm.state.broker.update( + if not self.charm.state.unit_broker.truststore_password: + self.charm.state.unit_broker.update( {"truststore-password": self.charm.workload.generate_password()} ) @@ -111,9 +113,9 @@ def _tls_relation_joined(self, _) -> None: def _tls_relation_broken(self, _) -> None: """Handler for `certificates_relation_broken` event.""" - self.charm.state.broker.update({"csr": ""}) - self.charm.state.broker.update({"certificate": ""}) - self.charm.state.broker.update({"ca": ""}) + self.charm.state.unit_broker.update( + {"csr": "", "certificate": "", "ca": "", "ca-cert": ""} + ) # remove all existing keystores from the unit so we don't preserve certs self.charm.tls_manager.remove_stores() @@ -141,22 +143,22 @@ def _trusted_relation_created(self, _) -> None: def _trusted_relation_joined(self, event: RelationJoinedEvent) -> None: """Generate a CSR so the tls-certificates operator works as expected.""" # Once the certificates have been added, TLS setup has finished - if not self.charm.state.broker.certificate: + if not self.charm.state.unit_broker.certificate: logger.debug("Missing TLS relation, deferring") event.defer() return alias = self.charm.tls_manager.generate_alias( - app_name=event.app.name, # pyright: ignore[reportOptionalMemberAccess] + app_name=event.app.name, relation_id=event.relation.id, ) - subject = os.uname()[1] if self.charm.substrate == "k8s" else self.charm.state.broker.host + subject = ( + os.uname()[1] if self.charm.substrate == "k8s" else self.charm.state.unit_broker.host + ) csr = ( generate_csr( add_unique_id_to_subject_name=bool(alias), - private_key=self.charm.state.broker.private_key.encode( # pyright: ignore[reportOptionalMemberAccess] - "utf-8" - ), + private_key=self.charm.state.unit_broker.private_key.encode("utf-8"), subject=subject, sans_ip=self._sans["sans_ip"], sans_dns=self._sans["sans_dns"], @@ -170,13 +172,16 @@ def _trusted_relation_joined(self, event: RelationJoinedEvent) -> None: def _trusted_relation_changed(self, event: RelationChangedEvent) -> None: """Overrides the requirer logic of TLSInterface.""" + if not event.relation or not event.relation.app: + return + # Once the certificates have been added, TLS setup has finished - if not self.charm.state.broker.certificate: + if not self.charm.state.unit_broker.certificate: logger.debug("Missing TLS relation, deferring") event.defer() return - relation_data = _load_relation_data(dict(event.relation.data[event.relation.app])) # type: ignore[reportOptionalMemberAccess] + relation_data = _load_relation_data(dict(event.relation.data[event.relation.app])) provider_certificates = relation_data.get("certificates", []) if not provider_certificates: @@ -185,7 +190,7 @@ def _trusted_relation_changed(self, event: RelationChangedEvent) -> None: return alias = self.charm.tls_manager.generate_alias( - event.relation.app.name, # pyright: ignore[reportOptionalMemberAccess] + event.relation.app.name, event.relation.id, ) # NOTE: Relation should only be used with one set of certificates, @@ -206,15 +211,18 @@ def _trusted_relation_changed(self, event: RelationChangedEvent) -> None: def _trusted_relation_broken(self, event: RelationBrokenEvent) -> None: """Handle relation broken for a trusted certificate/ca relation.""" + if not event.relation or not event.relation.app: + return + # Once the certificates have been added, TLS setup has finished - if not self.charm.state.broker.certificate: + if not self.charm.state.unit_broker.certificate: logger.debug("Missing TLS relation, deferring") event.defer() return # All units will need to remove the cert from their truststore alias = self.charm.tls_manager.generate_alias( - app_name=event.relation.app.name, # pyright: ignore[reportOptionalMemberAccess] + app_name=event.relation.app.name, relation_id=event.relation.id, ) @@ -245,12 +253,13 @@ def _on_certificate_available(self, event: CertificateAvailableEvent) -> None: return # avoid setting tls files and restarting - if event.certificate_signing_request != self.charm.state.broker.csr: + if event.certificate_signing_request != self.charm.state.unit_broker.csr: logger.error("Can't use certificate, found unknown CSR") return - self.charm.state.broker.update({"certificate": event.certificate}) - self.charm.state.broker.update({"ca": event.ca}) + self.charm.state.unit_broker.update( + {"certificate": event.certificate, "ca-cert": event.ca, "ca": ""} + ) self.charm.tls_manager.set_server_key() self.charm.tls_manager.set_ca() @@ -258,28 +267,32 @@ def _on_certificate_available(self, event: CertificateAvailableEvent) -> None: self.charm.tls_manager.set_truststore() self.charm.tls_manager.set_keystore() + # single-unit Kafka can lose restart events if it loses connection with TLS-enabled ZK + self.charm.on.config_changed.emit() + def _on_certificate_expiring(self, _) -> None: """Handler for `certificate_expiring` event.""" if ( - not self.charm.state.broker.private_key - or not self.charm.state.broker.csr + not self.charm.state.unit_broker.private_key + or not self.charm.state.unit_broker.csr or not self.charm.state.peer_relation ): logger.error("Missing unit private key and/or old csr") return + new_csr = generate_csr( - private_key=self.charm.state.broker.private_key.encode("utf-8"), - subject=self.charm.state.broker.relation_data.get("private-address", ""), + private_key=self.charm.state.unit_broker.private_key.encode("utf-8"), + subject=self.charm.state.unit_broker.relation_data.get("private-address", ""), sans_ip=self._sans["sans_ip"], sans_dns=self._sans["sans_dns"], ) self.certificates.request_certificate_renewal( - old_certificate_signing_request=self.charm.state.broker.csr.encode("utf-8"), + old_certificate_signing_request=self.charm.state.unit_broker.csr.encode("utf-8"), new_certificate_signing_request=new_csr, ) - self.charm.state.broker.update({"csr": new_csr.decode("utf-8").strip()}) + self.charm.state.unit_broker.update({"csr": new_csr.decode("utf-8").strip()}) def _set_tls_private_key(self, event: ActionEvent) -> None: """Handler for `set_tls_private_key` action.""" @@ -290,22 +303,22 @@ def _set_tls_private_key(self, event: ActionEvent) -> None: else base64.b64decode(key).decode("utf-8") ) - self.charm.state.broker.update({"private-key": private_key}) + self.charm.state.unit_broker.update({"private-key": private_key}) self._on_certificate_expiring(event) def _request_certificate(self): """Generates and submits CSR to provider.""" - if not self.charm.state.broker.private_key or not self.charm.state.peer_relation: + if not self.charm.state.unit_broker.private_key or not self.charm.state.peer_relation: logger.error("Can't request certificate, missing private key") return csr = generate_csr( - private_key=self.charm.state.broker.private_key.encode("utf-8"), - subject=self.charm.state.broker.relation_data.get("private-address", ""), + private_key=self.charm.state.unit_broker.private_key.encode("utf-8"), + subject=self.charm.state.unit_broker.relation_data.get("private-address", ""), sans_ip=self._sans["sans_ip"], sans_dns=self._sans["sans_dns"], ) - self.charm.state.broker.update({"csr": csr.decode("utf-8").strip()}) + self.charm.state.unit_broker.update({"csr": csr.decode("utf-8").strip()}) self.certificates.request_certificate_creation(certificate_signing_request=csr) @@ -314,7 +327,7 @@ def _sans(self) -> dict[str, list[str] | None]: """Builds a SAN dict of DNS names and IPs for the unit.""" if self.charm.substrate == "vm": return { - "sans_ip": [self.charm.state.broker.host], + "sans_ip": [self.charm.state.unit_broker.host], "sans_dns": [self.model.unit.name, socket.getfqdn()] + self._extra_sans, } else: @@ -325,8 +338,8 @@ def _sans(self) -> dict[str, list[str] | None]: return { "sans_ip": [str(bind_address)], "sans_dns": [ - self.charm.state.broker.host.split(".")[0], - self.charm.state.broker.host, + self.charm.state.unit_broker.host.split(".")[0], + self.charm.state.unit_broker.host, socket.getfqdn(), ] + self._extra_sans, diff --git a/src/events/zookeeper.py b/src/events/zookeeper.py index 5055fe8d..e43a791c 100644 --- a/src/events/zookeeper.py +++ b/src/events/zookeeper.py @@ -5,9 +5,10 @@ """Supporting objects for Kafka-Zookeeper relation.""" import logging -import subprocess +import subprocess # nosec B404 from typing import TYPE_CHECKING +from charms.data_platform_libs.v0.data_interfaces import DatabaseRequirerEventHandlers from ops import Object, RelationChangedEvent, RelationEvent from ops.pebble import ExecError @@ -25,6 +26,9 @@ class ZooKeeperHandler(Object): def __init__(self, charm) -> None: super().__init__(charm, "zookeeper_client") self.charm: "KafkaCharm" = charm + self.zookeeper_requires = DatabaseRequirerEventHandlers( + self.charm, self.charm.state.zookeeper_requires_interface + ) self.framework.observe(self.charm.on[ZK].relation_created, self._on_zookeeper_created) self.framework.observe(self.charm.on[ZK].relation_joined, self._on_zookeeper_changed) @@ -39,7 +43,6 @@ def _on_zookeeper_created(self, _) -> None: def _on_zookeeper_changed(self, event: RelationChangedEvent) -> None: """Handler for `zookeeper_relation_created/joined/changed` events, ensuring internal users get created.""" if not self.charm.state.zookeeper.zookeeper_connected: - logger.debug("No information found from ZooKeeper relation") self.charm._set_status(Status.ZK_NO_DATA) return @@ -51,7 +54,7 @@ def _on_zookeeper_changed(self, event: RelationChangedEvent) -> None: # do not create users until certificate + keystores created # otherwise unable to authenticate to ZK - if self.charm.state.cluster.tls_enabled and not self.charm.state.broker.certificate: + if self.charm.state.cluster.tls_enabled and not self.charm.state.unit_broker.certificate: self.charm._set_status(Status.NO_CERT) event.defer() return @@ -75,9 +78,9 @@ def _on_zookeeper_changed(self, event: RelationChangedEvent) -> None: # attempt re-start of Kafka for all units on zookeeper-changed # avoids relying on deferred events elsewhere that may not exist after cluster init if not self.charm.healthy and self.charm.state.cluster.internal_user_credentials: - self.charm._on_start(event) + self.charm.on.start.emit() - self.charm._on_config_changed(event) + self.charm.on.config_changed.emit() def _on_zookeeper_broken(self, _: RelationEvent) -> None: """Handler for `zookeeper_relation_broken` event, ensuring charm blocks.""" diff --git a/src/health.py b/src/health.py index 85f01ff2..49ba8cd7 100644 --- a/src/health.py +++ b/src/health.py @@ -56,7 +56,7 @@ def _get_partitions_size(self) -> tuple[int, int]: """Gets the number of partitions and their average size from the log dirs.""" log_dirs_command = [ "--describe", - f"--bootstrap-server {','.join(self.charm.state.bootstrap_server)}", + f"--bootstrap-server {self.charm.state.bootstrap_server}", f"--command-config {self.charm.workload.paths.client_properties}", ] try: diff --git a/src/literals.py b/src/literals.py index 803e3aa1..f947a8ba 100644 --- a/src/literals.py +++ b/src/literals.py @@ -14,29 +14,41 @@ SNAP_NAME = "charmed-kafka" CHARMED_KAFKA_SNAP_REVISION = 32 CONTAINER = "kafka" +SUBSTRATE = "vm" + +USER = "snap_daemon" +GROUP = "root" +# FIXME: these need better names PEER = "cluster" ZK = "zookeeper" REL_NAME = "kafka-client" -INTER_BROKER_USER = "sync" -ADMIN_USER = "admin" TLS_RELATION = "certificates" TRUSTED_CERTIFICATE_RELATION = "trusted-certificate" TRUSTED_CA_RELATION = "trusted-ca" + +INTER_BROKER_USER = "sync" +ADMIN_USER = "admin" INTERNAL_USERS = [INTER_BROKER_USER, ADMIN_USER] +SECRETS_APP = [f"{user}-password" for user in INTERNAL_USERS] +SECRETS_UNIT = [ + "ca-cert", + "csr", + "certificate", + "truststore-password", + "keystore-password", + "private-key", +] + JMX_EXPORTER_PORT = 9101 METRICS_RULES_DIR = "./src/alert_rules/prometheus" LOGS_RULES_DIR = "./src/alert_rules/loki" -SUBSTRATE = "vm" -USER = "snap_daemon" -GROUP = "root" - AuthMechanism = Literal["SASL_PLAINTEXT", "SASL_SSL", "SSL"] Scope = Literal["INTERNAL", "CLIENT"] DebugLevel = Literal["DEBUG", "INFO", "WARNING", "ERROR"] -Substrate = Literal["vm", "k8s"] DatabagScope = Literal["unit", "app"] +Substrates = Literal["vm", "k8s"] JVM_MEM_MIN_GB = 1 JVM_MEM_MAX_GB = 6 diff --git a/src/managers/auth.py b/src/managers/auth.py index 74a700ec..fd139781 100644 --- a/src/managers/auth.py +++ b/src/managers/auth.py @@ -6,7 +6,7 @@ import logging import re -import subprocess +import subprocess # nosec B404 from dataclasses import asdict, dataclass from ops.pebble import ExecError @@ -34,12 +34,6 @@ def __init__(self, state: ClusterState, workload: WorkloadBase, kafka_opts: str) self.state = state self.workload = workload self.kafka_opts = kafka_opts - - self.zookeeper_connect = self.state.zookeeper.connect - self.bootstrap_server = ",".join(self.state.bootstrap_server) - self.client_properties = self.workload.paths.client_properties - self.server_properties = self.workload.paths.server_properties - self.new_user_acls: set[Acl] = set() @property @@ -51,8 +45,8 @@ def current_acls(self) -> set[Acl]: def _get_acls_from_cluster(self) -> str: """Loads the currently active ACLs from the Kafka cluster.""" command = [ - f"--bootstrap-server={self.bootstrap_server}", - f"--command-config={self.client_properties}", + f"--bootstrap-server={self.state.bootstrap_server}", + f"--command-config={self.workload.paths.client_properties}", "--list", ] acls = self.workload.run_bin_command(bin_keyword="acls", bin_args=command) @@ -160,14 +154,14 @@ def add_user(self, username: str, password: str, zk_auth: bool = False) -> None: # instead must be authorized using ZooKeeper JAAS if zk_auth: command = base_command + [ - f"--zookeeper={self.zookeeper_connect}", - f"--zk-tls-config-file={self.server_properties}", + f"--zookeeper={self.state.zookeeper.connect}", + f"--zk-tls-config-file={self.workload.paths.server_properties}", ] opts = [self.kafka_opts] else: command = base_command + [ - f"--bootstrap-server={self.bootstrap_server}", - f"--command-config={self.client_properties}", + f"--bootstrap-server={self.state.bootstrap_server}", + f"--command-config={self.workload.paths.client_properties}", ] opts = [] @@ -183,8 +177,8 @@ def delete_user(self, username: str) -> None: `(subprocess.CalledProcessError | ops.pebble.ExecError)`: if the error returned a non-zero exit code """ command = [ - f"--bootstrap-server={self.bootstrap_server}", - f"--command-config={self.client_properties}", + f"--bootstrap-server={self.state.bootstrap_server}", + f"--command-config={self.workload.paths.client_properties}", "--alter", "--entity-type=users", f"--entity-name={username}", @@ -218,8 +212,8 @@ def add_acl( `(subprocess.CalledProcessError | ops.pebble.ExecError)`: if the error returned a non-zero exit code """ command = [ - f"--bootstrap-server={self.bootstrap_server}", - f"--command-config={self.client_properties}", + f"--bootstrap-server={self.state.bootstrap_server}", + f"--command-config={self.workload.paths.client_properties}", "--add", f"--allow-principal=User:{username}", f"--operation={operation}", @@ -251,8 +245,8 @@ def remove_acl( `(subprocess.CalledProcessError | ops.pebble.ExecError)`: if the error returned a non-zero exit code """ command = [ - f"--bootstrap-server={self.bootstrap_server}", - f"--command-config={self.client_properties}", + f"--bootstrap-server={self.state.bootstrap_server}", + f"--command-config={self.workload.paths.client_properties}", "--remove", f"--allow-principal=User:{username}", f"--operation={operation}", diff --git a/src/managers/config.py b/src/managers/config.py index 2944d0cc..0c5cb5f4 100644 --- a/src/managers/config.py +++ b/src/managers/config.py @@ -212,7 +212,7 @@ def auth_properties(self) -> list[str]: List of properties to be set """ return [ - f"broker.id={self.state.broker.unit_id}", + f"broker.id={self.state.unit_broker.unit_id}", f"zookeeper.connect={self.state.zookeeper.connect}", ] @@ -226,7 +226,7 @@ def zookeeper_tls_properties(self) -> list[str]: return [ "zookeeper.ssl.client.enable=true", f"zookeeper.ssl.truststore.location={self.workload.paths.truststore}", - f"zookeeper.ssl.truststore.password={self.state.broker.truststore_password}", + f"zookeeper.ssl.truststore.password={self.state.unit_broker.truststore_password}", "zookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty", ] @@ -240,9 +240,9 @@ def tls_properties(self) -> list[str]: mtls = "required" if self.state.cluster.mtls_enabled else "none" return [ f"ssl.truststore.location={self.workload.paths.truststore}", - f"ssl.truststore.password={self.state.broker.truststore_password}", + f"ssl.truststore.password={self.state.unit_broker.truststore_password}", f"ssl.keystore.location={self.workload.paths.keystore}", - f"ssl.keystore.password={self.state.broker.keystore_password}", + f"ssl.keystore.password={self.state.unit_broker.keystore_password}", f"ssl.client.auth={mtls}", ] @@ -275,7 +275,7 @@ def security_protocol(self) -> AuthMechanism: # FIXME: When we have multiple auth_mechanims/listeners, remove this method return ( "SASL_SSL" - if (self.state.cluster.tls_enabled and self.state.broker.certificate) + if (self.state.cluster.tls_enabled and self.state.unit_broker.certificate) else "SASL_PLAINTEXT" ) @@ -294,7 +294,7 @@ def auth_mechanisms(self) -> list[AuthMechanism]: def internal_listener(self) -> Listener: """Return the internal listener.""" protocol = self.security_protocol - return Listener(host=self.state.broker.host, protocol=protocol, scope="INTERNAL") + return Listener(host=self.state.unit_broker.host, protocol=protocol, scope="INTERNAL") @property def client_listeners(self) -> list[Listener]: @@ -304,7 +304,7 @@ def client_listeners(self) -> list[Listener]: return [] return [ - Listener(host=self.state.broker.host, protocol=auth, scope="CLIENT") + Listener(host=self.state.unit_broker.host, protocol=auth, scope="CLIENT") for auth in self.auth_mechanisms ] @@ -352,10 +352,10 @@ def client_properties(self) -> list[str]: "sasl.mechanism=SCRAM-SHA-512", f"security.protocol={self.security_protocol}", # FIXME: security.protocol will need changing once multiple listener auth schemes - f"bootstrap.servers={','.join(self.state.bootstrap_server)}", + f"bootstrap.servers={self.state.bootstrap_server}", ] - if self.state.cluster.tls_enabled and self.state.broker.certificate: + if self.state.cluster.tls_enabled and self.state.unit_broker.certificate: client_properties += self.tls_properties return client_properties @@ -394,7 +394,7 @@ def server_properties(self) -> list[str]: + DEFAULT_CONFIG_OPTIONS.split("\n") ) - if self.state.cluster.tls_enabled and self.state.broker.certificate: + if self.state.cluster.tls_enabled and self.state.unit_broker.certificate: properties += self.tls_properties + self.zookeeper_tls_properties return properties diff --git a/src/managers/tls.py b/src/managers/tls.py index 389fbd63..2b950afc 100644 --- a/src/managers/tls.py +++ b/src/managers/tls.py @@ -5,13 +5,13 @@ """Manager for handling Kafka TLS configuration.""" import logging -import subprocess +import subprocess # nosec B404 from ops.pebble import ExecError from core.cluster import ClusterState from core.workload import WorkloadBase -from literals import GROUP, USER, Substrate +from literals import GROUP, USER, Substrates logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ class TLSManager: """Manager for building necessary files for Java TLS auth.""" - def __init__(self, state: ClusterState, workload: WorkloadBase, substrate: Substrate): + def __init__(self, state: ClusterState, workload: WorkloadBase, substrate: Substrates): self.state = state self.workload = workload self.substrate = substrate @@ -32,39 +32,39 @@ def generate_alias(self, app_name: str, relation_id: int) -> str: def set_server_key(self) -> None: """Sets the unit private-key.""" - if not self.state.broker.private_key: + if not self.state.unit_broker.private_key: logger.error("Can't set private-key to unit, missing private-key in relation data") return self.workload.write( - content=self.state.broker.private_key, + content=self.state.unit_broker.private_key, path=f"{self.workload.paths.conf_path}/server.key", ) def set_ca(self) -> None: """Sets the unit ca.""" - if not self.state.broker.ca: + if not self.state.unit_broker.ca: logger.error("Can't set CA to unit, missing CA in relation data") return self.workload.write( - content=self.state.broker.ca, path=f"{self.workload.paths.conf_path}/ca.pem" + content=self.state.unit_broker.ca, path=f"{self.workload.paths.conf_path}/ca.pem" ) def set_certificate(self) -> None: """Sets the unit certificate.""" - if not self.state.broker.certificate: + if not self.state.unit_broker.certificate: logger.error("Can't set certificate to unit, missing certificate in relation data") return self.workload.write( - content=self.state.broker.certificate, + content=self.state.unit_broker.certificate, path=f"{self.workload.paths.conf_path}/server.pem", ) def set_truststore(self) -> None: """Adds CA to JKS truststore.""" - command = f"{self.keytool} -import -v -alias ca -file ca.pem -keystore truststore.jks -storepass {self.state.broker.truststore_password} -noprompt" + command = f"{self.keytool} -import -v -alias ca -file ca.pem -keystore truststore.jks -storepass {self.state.unit_broker.truststore_password} -noprompt" try: self.workload.exec(command=command, working_dir=self.workload.paths.conf_path) self.workload.exec(f"chown {USER}:{GROUP} {self.workload.paths.truststore}") @@ -78,7 +78,7 @@ def set_truststore(self) -> None: def set_keystore(self) -> None: """Creates and adds unit cert and private-key to the keystore.""" - command = f"openssl pkcs12 -export -in server.pem -inkey server.key -passin pass:{self.state.broker.keystore_password} -certfile server.pem -out keystore.p12 -password pass:{self.state.broker.keystore_password}" + command = f"openssl pkcs12 -export -in server.pem -inkey server.key -passin pass:{self.state.unit_broker.keystore_password} -certfile server.pem -out keystore.p12 -password pass:{self.state.unit_broker.keystore_password}" try: self.workload.exec(command=command, working_dir=self.workload.paths.conf_path) self.workload.exec(f"chown {USER}:{GROUP} {self.workload.paths.keystore}") @@ -89,7 +89,7 @@ def set_keystore(self) -> None: def import_cert(self, alias: str, filename: str) -> None: """Add a certificate to the truststore.""" - command = f"{self.keytool} -import -v -alias {alias} -file {filename} -keystore truststore.jks -storepass {self.state.broker.truststore_password} -noprompt" + command = f"{self.keytool} -import -v -alias {alias} -file {filename} -keystore truststore.jks -storepass {self.state.unit_broker.truststore_password} -noprompt" try: self.workload.exec(command=command, working_dir=self.workload.paths.conf_path) except (subprocess.CalledProcessError, ExecError) as e: @@ -103,7 +103,7 @@ def import_cert(self, alias: str, filename: str) -> None: def remove_cert(self, alias: str) -> None: """Remove a cert from the truststore.""" try: - command = f"{self.keytool} -delete -v -alias {alias} -keystore truststore.jks -storepass {self.state.broker.truststore_password} -noprompt" + command = f"{self.keytool} -delete -v -alias {alias} -keystore truststore.jks -storepass {self.state.unit_broker.truststore_password} -noprompt" self.workload.exec(command=command, working_dir=self.workload.paths.conf_path) self.workload.exec(f"rm -f {alias}.pem", working_dir=self.workload.paths.conf_path) except (subprocess.CalledProcessError, ExecError) as e: diff --git a/src/workload.py b/src/workload.py index ba5bca43..95db518a 100644 --- a/src/workload.py +++ b/src/workload.py @@ -82,6 +82,7 @@ def exec( stderr=subprocess.PIPE, universal_newlines=True, shell=True, + env=env, cwd=working_dir, ) logger.debug(f"{output=}") diff --git a/tests/integration/app-charm/lib/charms/data_platform_libs/v0/data_interfaces.py b/tests/integration/app-charm/lib/charms/data_platform_libs/v0/data_interfaces.py index 636cace7..3ce69e15 100644 --- a/tests/integration/app-charm/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/tests/integration/app-charm/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Library to manage the relation for the data-platform products. +r"""Library to manage the relation for the data-platform products. This library contains the Requires and Provides classes for handling the relation between an application and multiple managed application supported by the data-team: @@ -144,6 +144,19 @@ def _on_cluster2_database_created(self, event: DatabaseCreatedEvent) -> None: ``` +When it's needed to check whether a plugin (extension) is enabled on the PostgreSQL +charm, you can use the is_postgresql_plugin_enabled method. To use that, you need to +add the following dependency to your charmcraft.yaml file: + +```yaml + +parts: + charm: + charm-binary-python-packages: + - psycopg[binary] + +``` + ### Provider Charm Following an example of using the DatabaseRequestedEvent, in the context of the @@ -278,22 +291,37 @@ def _on_topic_requested(self, event: TopicRequestedEvent): exchanged in the relation databag. """ +import copy import json import logging from abc import ABC, abstractmethod -from collections import namedtuple +from collections import UserDict, namedtuple from datetime import datetime -from typing import List, Optional +from enum import Enum +from typing import ( + Callable, + Dict, + ItemsView, + KeysView, + List, + Optional, + Set, + Tuple, + Union, + ValuesView, +) +from ops import JujuVersion, Model, Secret, SecretInfo, SecretNotFoundError from ops.charm import ( CharmBase, CharmEvents, RelationChangedEvent, + RelationCreatedEvent, RelationEvent, - RelationJoinedEvent, + SecretChangedEvent, ) from ops.framework import EventSource, Object -from ops.model import Relation +from ops.model import Application, ModelError, Relation, Unit # The unique Charmhub library identifier, never change it LIBID = "6c3e6b6680d64e9c89e611d1a15f65be" @@ -303,7 +331,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 9 +LIBPATCH = 34 PYDEPS = ["ops>=2.0.0"] @@ -318,7 +346,98 @@ def _on_topic_requested(self, event: TopicRequestedEvent): deleted - key that were deleted""" -def diff(event: RelationChangedEvent, bucket: str) -> Diff: +PROV_SECRET_PREFIX = "secret-" +REQ_SECRET_FIELDS = "requested-secrets" +GROUP_MAPPING_FIELD = "secret_group_mapping" +GROUP_SEPARATOR = "@" + + +class SecretGroup(str): + """Secret groups specific type.""" + + +class SecretGroupsAggregate(str): + """Secret groups with option to extend with additional constants.""" + + def __init__(self): + self.USER = SecretGroup("user") + self.TLS = SecretGroup("tls") + self.EXTRA = SecretGroup("extra") + + def __setattr__(self, name, value): + """Setting internal constants.""" + if name in self.__dict__: + raise RuntimeError("Can't set constant!") + else: + super().__setattr__(name, SecretGroup(value)) + + def groups(self) -> list: + """Return the list of stored SecretGroups.""" + return list(self.__dict__.values()) + + def get_group(self, group: str) -> Optional[SecretGroup]: + """If the input str translates to a group name, return that.""" + return SecretGroup(group) if group in self.groups() else None + + +SECRET_GROUPS = SecretGroupsAggregate() + + +class DataInterfacesError(Exception): + """Common ancestor for DataInterfaces related exceptions.""" + + +class SecretError(DataInterfacesError): + """Common ancestor for Secrets related exceptions.""" + + +class SecretAlreadyExistsError(SecretError): + """A secret that was to be added already exists.""" + + +class SecretsUnavailableError(SecretError): + """Secrets aren't yet available for Juju version used.""" + + +class SecretsIllegalUpdateError(SecretError): + """Secrets aren't yet available for Juju version used.""" + + +class IllegalOperationError(DataInterfacesError): + """To be used when an operation is not allowed to be performed.""" + + +def get_encoded_dict( + relation: Relation, member: Union[Unit, Application], field: str +) -> Optional[Dict[str, str]]: + """Retrieve and decode an encoded field from relation data.""" + data = json.loads(relation.data[member].get(field, "{}")) + if isinstance(data, dict): + return data + logger.error("Unexpected datatype for %s instead of dict.", str(data)) + + +def get_encoded_list( + relation: Relation, member: Union[Unit, Application], field: str +) -> Optional[List[str]]: + """Retrieve and decode an encoded field from relation data.""" + data = json.loads(relation.data[member].get(field, "[]")) + if isinstance(data, list): + return data + logger.error("Unexpected datatype for %s instead of list.", str(data)) + + +def set_encoded_field( + relation: Relation, + member: Union[Unit, Application], + field: str, + value: Union[str, list, Dict[str, str]], +) -> None: + """Set an encoded field from relation data.""" + relation.data[member].update({field: json.dumps(value)}) + + +def diff(event: RelationChangedEvent, bucket: Optional[Union[Unit, Application]]) -> Diff: """Retrieves the diff of the data in the relation changed databag. Args: @@ -330,302 +449,1940 @@ def diff(event: RelationChangedEvent, bucket: str) -> Diff: keys from the event relation databag. """ # Retrieve the old data from the data key in the application relation databag. - old_data = json.loads(event.relation.data[bucket].get("data", "{}")) + if not bucket: + return Diff([], [], []) + + old_data = get_encoded_dict(event.relation, bucket, "data") + + if not old_data: + old_data = {} + # Retrieve the new data from the event relation databag. - new_data = { - key: value for key, value in event.relation.data[event.app].items() if key != "data" - } + new_data = ( + {key: value for key, value in event.relation.data[event.app].items() if key != "data"} + if event.app + else {} + ) # These are the keys that were added to the databag and triggered this event. - added = new_data.keys() - old_data.keys() + added = new_data.keys() - old_data.keys() # pyright: ignore [reportAssignmentType] # These are the keys that were removed from the databag and triggered this event. - deleted = old_data.keys() - new_data.keys() + deleted = old_data.keys() - new_data.keys() # pyright: ignore [reportAssignmentType] # These are the keys that already existed in the databag, # but had their values changed. - changed = {key for key in old_data.keys() & new_data.keys() if old_data[key] != new_data[key]} + changed = { + key + for key in old_data.keys() & new_data.keys() # pyright: ignore [reportAssignmentType] + if old_data[key] != new_data[key] # pyright: ignore [reportAssignmentType] + } # Convert the new_data to a serializable format and save it for a next diff check. - event.relation.data[bucket].update({"data": json.dumps(new_data)}) + set_encoded_field(event.relation, bucket, "data", new_data) # Return the diff with all possible changes. return Diff(added, changed, deleted) -# Base DataProvides and DataRequires +def leader_only(f): + """Decorator to ensure that only leader can perform given operation.""" + def wrapper(self, *args, **kwargs): + if self.component == self.local_app and not self.local_unit.is_leader(): + logger.error( + "This operation (%s()) can only be performed by the leader unit", f.__name__ + ) + return + return f(self, *args, **kwargs) -class DataProvides(Object, ABC): - """Base provides-side of the data products relation.""" - - def __init__(self, charm: CharmBase, relation_name: str) -> None: - super().__init__(charm, relation_name) - self.charm = charm - self.local_app = self.charm.model.app - self.local_unit = self.charm.unit - self.relation_name = relation_name - self.framework.observe( - charm.on[relation_name].relation_changed, - self._on_relation_changed, - ) + wrapper.leader_only = True + return wrapper - def _diff(self, event: RelationChangedEvent) -> Diff: - """Retrieves the diff of the data in the relation changed databag. - Args: - event: relation changed event. +def juju_secrets_only(f): + """Decorator to ensure that certain operations would be only executed on Juju3.""" - Returns: - a Diff instance containing the added, deleted and changed - keys from the event relation databag. - """ - return diff(event, self.local_app) + def wrapper(self, *args, **kwargs): + if not self.secrets_enabled: + raise SecretsUnavailableError("Secrets unavailable on current Juju version") + return f(self, *args, **kwargs) - @abstractmethod - def _on_relation_changed(self, event: RelationChangedEvent) -> None: - """Event emitted when the relation data has changed.""" - raise NotImplementedError + return wrapper - def fetch_relation_data(self) -> dict: - """Retrieves data from relation. - This function can be used to retrieve data from a relation - in the charm code when outside an event callback. +def dynamic_secrets_only(f): + """Decorator to ensure that certain operations would be only executed when NO static secrets are defined.""" - Returns: - a dict of the values stored in the relation data bag - for all relation instances (indexed by the relation id). - """ - data = {} - for relation in self.relations: - data[relation.id] = { - key: value for key, value in relation.data[relation.app].items() if key != "data" - } - return data + def wrapper(self, *args, **kwargs): + if self.static_secret_fields: + raise IllegalOperationError( + "Unsafe usage of statically and dynamically defined secrets, aborting." + ) + return f(self, *args, **kwargs) - def _update_relation_data(self, relation_id: int, data: dict) -> None: - """Updates a set of key-value pairs in the relation. + return wrapper - This function writes in the application data bag, therefore, - only the leader unit can call it. - Args: - relation_id: the identifier for a particular relation. - data: dict containing the key-value pairs - that should be updated in the relation. - """ - if self.local_unit.is_leader(): - relation = self.charm.model.get_relation(self.relation_name, relation_id) - relation.data[self.local_app].update(data) +def either_static_or_dynamic_secrets(f): + """Decorator to ensure that static and dynamic secrets won't be used in parallel.""" - @property - def relations(self) -> List[Relation]: - """The list of Relation instances associated with this relation_name.""" - return list(self.charm.model.relations[self.relation_name]) + def wrapper(self, *args, **kwargs): + if self.static_secret_fields and set(self.current_secret_fields) - set( + self.static_secret_fields + ): + raise IllegalOperationError( + "Unsafe usage of statically and dynamically defined secrets, aborting." + ) + return f(self, *args, **kwargs) - def set_credentials(self, relation_id: int, username: str, password: str) -> None: - """Set credentials. + return wrapper - This function writes in the application data bag, therefore, - only the leader unit can call it. - Args: - relation_id: the identifier for a particular relation. - username: user that was created. - password: password of the created user. - """ - self._update_relation_data( - relation_id, - { - "username": username, - "password": password, - }, - ) +class Scope(Enum): + """Peer relations scope.""" - def set_tls(self, relation_id: int, tls: str) -> None: - """Set whether TLS is enabled. + APP = "app" + UNIT = "unit" - Args: - relation_id: the identifier for a particular relation. - tls: whether tls is enabled (True or False). - """ - self._update_relation_data(relation_id, {"tls": tls}) - def set_tls_ca(self, relation_id: int, tls_ca: str) -> None: - """Set the TLS CA in the application relation databag. +################################################################################ +# Secrets internal caching +################################################################################ - Args: - relation_id: the identifier for a particular relation. - tls_ca: TLS certification authority. - """ - self._update_relation_data(relation_id, {"tls_ca": tls_ca}) +class CachedSecret: + """Locally cache a secret. -class DataRequires(Object, ABC): - """Requires-side of the relation.""" + The data structure is precisely re-using/simulating as in the actual Secret Storage + """ def __init__( self, - charm, - relation_name: str, - extra_user_roles: str = None, + model: Model, + component: Union[Application, Unit], + label: str, + secret_uri: Optional[str] = None, + legacy_labels: List[str] = [], ): - """Manager of base client relations.""" - super().__init__(charm, relation_name) - self.charm = charm - self.extra_user_roles = extra_user_roles - self.local_app = self.charm.model.app - self.local_unit = self.charm.unit - self.relation_name = relation_name - self.framework.observe( - self.charm.on[relation_name].relation_joined, self._on_relation_joined_event - ) - self.framework.observe( - self.charm.on[relation_name].relation_changed, self._on_relation_changed_event - ) + self._secret_meta = None + self._secret_content = {} + self._secret_uri = secret_uri + self.label = label + self._model = model + self.component = component + self.legacy_labels = legacy_labels + self.current_label = None + + def add_secret( + self, + content: Dict[str, str], + relation: Optional[Relation] = None, + label: Optional[str] = None, + ) -> Secret: + """Create a new secret.""" + if self._secret_uri: + raise SecretAlreadyExistsError( + "Secret is already defined with uri %s", self._secret_uri + ) - @abstractmethod - def _on_relation_joined_event(self, event: RelationJoinedEvent) -> None: - """Event emitted when the application joins the relation.""" - raise NotImplementedError + label = self.label if not label else label - @abstractmethod - def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: - raise NotImplementedError + secret = self.component.add_secret(content, label=label) + if relation and relation.app != self._model.app: + # If it's not a peer relation, grant is to be applied + secret.grant(relation) + self._secret_uri = secret.id + self._secret_meta = secret + return self._secret_meta - def fetch_relation_data(self) -> dict: - """Retrieves data from relation. + @property + def meta(self) -> Optional[Secret]: + """Getting cached secret meta-information.""" + if not self._secret_meta: + if not (self._secret_uri or self.label): + return + + for label in [self.label] + self.legacy_labels: + try: + self._secret_meta = self._model.get_secret(label=label) + except SecretNotFoundError: + pass + else: + if label != self.label: + self.current_label = label + break + + # If still not found, to be checked by URI, to be labelled with the proposed label + if not self._secret_meta and self._secret_uri: + self._secret_meta = self._model.get_secret(id=self._secret_uri, label=self.label) + return self._secret_meta + + def get_content(self) -> Dict[str, str]: + """Getting cached secret content.""" + if not self._secret_content: + if self.meta: + try: + self._secret_content = self.meta.get_content(refresh=True) + except (ValueError, ModelError) as err: + # https://bugs.launchpad.net/juju/+bug/2042596 + # Only triggered when 'refresh' is set + known_model_errors = [ + "ERROR either URI or label should be used for getting an owned secret but not both", + "ERROR secret owner cannot use --refresh", + ] + if isinstance(err, ModelError) and not any( + msg in str(err) for msg in known_model_errors + ): + raise + # Due to: ValueError: Secret owner cannot use refresh=True + self._secret_content = self.meta.get_content() + return self._secret_content + + def _move_to_new_label_if_needed(self): + """Helper function to re-create the secret with a different label.""" + if not self.current_label or not (self.meta and self._secret_meta): + return - This function can be used to retrieve data from a relation - in the charm code when outside an event callback. - Function cannot be used in `*-relation-broken` events and will raise an exception. + # Create a new secret with the new label + old_meta = self._secret_meta + content = self._secret_meta.get_content() - Returns: - a dict of the values stored in the relation data bag - for all relation instances (indexed by the relation ID). - """ - data = {} - for relation in self.relations: - data[relation.id] = { - key: value for key, value in relation.data[relation.app].items() if key != "data" - } - return data + # I wish we could just check if we are the owners of the secret... + try: + self._secret_meta = self.add_secret(content, label=self.label) + except ModelError as err: + if "this unit is not the leader" not in str(err): + raise + old_meta.remove_all_revisions() + + def set_content(self, content: Dict[str, str]) -> None: + """Setting cached secret content.""" + if not self.meta: + return - def _update_relation_data(self, relation_id: int, data: dict) -> None: - """Updates a set of key-value pairs in the relation. + if content: + self._move_to_new_label_if_needed() + self.meta.set_content(content) + self._secret_content = content + else: + self.meta.remove_all_revisions() - This function writes in the application data bag, therefore, - only the leader unit can call it. + def get_info(self) -> Optional[SecretInfo]: + """Wrapper function to apply the corresponding call on the Secret object within CachedSecret if any.""" + if self.meta: + return self.meta.get_info() - Args: - relation_id: the identifier for a particular relation. - data: dict containing the key-value pairs - that should be updated in the relation. - """ - if self.local_unit.is_leader(): - relation = self.charm.model.get_relation(self.relation_name, relation_id) - relation.data[self.local_app].update(data) + def remove(self) -> None: + """Remove secret.""" + if not self.meta: + raise SecretsUnavailableError("Non-existent secret was attempted to be removed.") + try: + self.meta.remove_all_revisions() + except SecretNotFoundError: + pass + self._secret_content = {} + self._secret_meta = None + self._secret_uri = None + + +class SecretCache: + """A data structure storing CachedSecret objects.""" + + def __init__(self, model: Model, component: Union[Application, Unit]): + self._model = model + self.component = component + self._secrets: Dict[str, CachedSecret] = {} + + def get( + self, label: str, uri: Optional[str] = None, legacy_labels: List[str] = [] + ) -> Optional[CachedSecret]: + """Getting a secret from Juju Secret store or cache.""" + if not self._secrets.get(label): + secret = CachedSecret( + self._model, self.component, label, uri, legacy_labels=legacy_labels + ) + if secret.meta: + self._secrets[label] = secret + return self._secrets.get(label) + + def add(self, label: str, content: Dict[str, str], relation: Relation) -> CachedSecret: + """Adding a secret to Juju Secret.""" + if self._secrets.get(label): + raise SecretAlreadyExistsError(f"Secret {label} already exists") + + secret = CachedSecret(self._model, self.component, label) + secret.add_secret(content, relation) + self._secrets[label] = secret + return self._secrets[label] + + def remove(self, label: str) -> None: + """Remove a secret from the cache.""" + if secret := self.get(label): + try: + secret.remove() + self._secrets.pop(label) + except (SecretsUnavailableError, KeyError): + pass + else: + return + logging.debug("Non-existing Juju Secret was attempted to be removed %s", label) - def _diff(self, event: RelationChangedEvent) -> Diff: - """Retrieves the diff of the data in the relation changed databag. - Args: - event: relation changed event. +################################################################################ +# Relation Data base/abstract ancestors (i.e. parent classes) +################################################################################ + + +# Base Data + + +class DataDict(UserDict): + """Python Standard Library 'dict' - like representation of Relation Data.""" + + def __init__(self, relation_data: "Data", relation_id: int): + self.relation_data = relation_data + self.relation_id = relation_id + + @property + def data(self) -> Dict[str, str]: + """Return the full content of the Abstract Relation Data dictionary.""" + result = self.relation_data.fetch_my_relation_data([self.relation_id]) + try: + result_remote = self.relation_data.fetch_relation_data([self.relation_id]) + except NotImplementedError: + result_remote = {self.relation_id: {}} + if result: + result_remote[self.relation_id].update(result[self.relation_id]) + return result_remote.get(self.relation_id, {}) + + def __setitem__(self, key: str, item: str) -> None: + """Set an item of the Abstract Relation Data dictionary.""" + self.relation_data.update_relation_data(self.relation_id, {key: item}) + + def __getitem__(self, key: str) -> str: + """Get an item of the Abstract Relation Data dictionary.""" + result = None + + # Avoiding "leader_only" error when cross-charm non-leader unit, not to report useless error + if ( + not hasattr(self.relation_data.fetch_my_relation_field, "leader_only") + or self.relation_data.component != self.relation_data.local_app + or self.relation_data.local_unit.is_leader() + ): + result = self.relation_data.fetch_my_relation_field(self.relation_id, key) + + if not result: + try: + result = self.relation_data.fetch_relation_field(self.relation_id, key) + except NotImplementedError: + pass + + if not result: + raise KeyError + return result + + def __eq__(self, d: dict) -> bool: + """Equality.""" + return self.data == d + + def __repr__(self) -> str: + """String representation Abstract Relation Data dictionary.""" + return repr(self.data) + + def __len__(self) -> int: + """Length of the Abstract Relation Data dictionary.""" + return len(self.data) + + def __delitem__(self, key: str) -> None: + """Delete an item of the Abstract Relation Data dictionary.""" + self.relation_data.delete_relation_data(self.relation_id, [key]) + + def has_key(self, key: str) -> bool: + """Does the key exist in the Abstract Relation Data dictionary?""" + return key in self.data + + def update(self, items: Dict[str, str]): + """Update the Abstract Relation Data dictionary.""" + self.relation_data.update_relation_data(self.relation_id, items) + + def keys(self) -> KeysView[str]: + """Keys of the Abstract Relation Data dictionary.""" + return self.data.keys() + + def values(self) -> ValuesView[str]: + """Values of the Abstract Relation Data dictionary.""" + return self.data.values() + + def items(self) -> ItemsView[str, str]: + """Items of the Abstract Relation Data dictionary.""" + return self.data.items() + + def pop(self, item: str) -> str: + """Pop an item of the Abstract Relation Data dictionary.""" + result = self.relation_data.fetch_my_relation_field(self.relation_id, item) + if not result: + raise KeyError(f"Item {item} doesn't exist.") + self.relation_data.delete_relation_data(self.relation_id, [item]) + return result + + def __contains__(self, item: str) -> bool: + """Does the Abstract Relation Data dictionary contain item?""" + return item in self.data.values() + + def __iter__(self): + """Iterate through the Abstract Relation Data dictionary.""" + return iter(self.data) + + def get(self, key: str, default: Optional[str] = None) -> Optional[str]: + """Safely get an item of the Abstract Relation Data dictionary.""" + try: + if result := self[key]: + return result + except KeyError: + return default - Returns: - a Diff instance containing the added, deleted and changed - keys from the event relation databag. - """ - return diff(event, self.local_unit) + +class Data(ABC): + """Base relation data mainpulation (abstract) class.""" + + SCOPE = Scope.APP + + # Local map to associate mappings with secrets potentially as a group + SECRET_LABEL_MAP = { + "username": SECRET_GROUPS.USER, + "password": SECRET_GROUPS.USER, + "uris": SECRET_GROUPS.USER, + "tls": SECRET_GROUPS.TLS, + "tls-ca": SECRET_GROUPS.TLS, + } + + def __init__( + self, + model: Model, + relation_name: str, + ) -> None: + self._model = model + self.local_app = self._model.app + self.local_unit = self._model.unit + self.relation_name = relation_name + self._jujuversion = None + self.component = self.local_app if self.SCOPE == Scope.APP else self.local_unit + self.secrets = SecretCache(self._model, self.component) + self.data_component = None @property def relations(self) -> List[Relation]: """The list of Relation instances associated with this relation_name.""" return [ relation - for relation in self.charm.model.relations[self.relation_name] + for relation in self._model.relations[self.relation_name] if self._is_relation_active(relation) ] + @property + def secrets_enabled(self): + """Is this Juju version allowing for Secrets usage?""" + if not self._jujuversion: + self._jujuversion = JujuVersion.from_environ() + return self._jujuversion.has_secrets + + @property + def secret_label_map(self): + """Exposing secret-label map via a property -- could be overridden in descendants!""" + return self.SECRET_LABEL_MAP + + # Mandatory overrides for internal/helper methods + + @abstractmethod + def _get_relation_secret( + self, relation_id: int, group_mapping: SecretGroup, relation_name: Optional[str] = None + ) -> Optional[CachedSecret]: + """Retrieve a Juju Secret that's been stored in the relation databag.""" + raise NotImplementedError + + @abstractmethod + def _fetch_specific_relation_data( + self, relation: Relation, fields: Optional[List[str]] + ) -> Dict[str, str]: + """Fetch data available (directily or indirectly -- i.e. secrets) from the relation.""" + raise NotImplementedError + + @abstractmethod + def _fetch_my_specific_relation_data( + self, relation: Relation, fields: Optional[List[str]] + ) -> Dict[str, str]: + """Fetch data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + raise NotImplementedError + + @abstractmethod + def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: + """Update data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + raise NotImplementedError + + @abstractmethod + def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: + """Delete data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + raise NotImplementedError + + # Internal helper methods + @staticmethod def _is_relation_active(relation: Relation): + """Whether the relation is active based on contained data.""" try: _ = repr(relation.data) return True - except RuntimeError: + except (RuntimeError, ModelError): return False @staticmethod - def _is_resource_created_for_relation(relation: Relation): - return ( - "username" in relation.data[relation.app] and "password" in relation.data[relation.app] - ) + def _is_secret_field(field: str) -> bool: + """Is the field in question a secret reference (URI) field or not?""" + return field.startswith(PROV_SECRET_PREFIX) - def is_resource_created(self, relation_id: Optional[int] = None) -> bool: - """Check if the resource has been created. + @staticmethod + def _generate_secret_label( + relation_name: str, relation_id: int, group_mapping: SecretGroup + ) -> str: + """Generate unique group_mappings for secrets within a relation context.""" + return f"{relation_name}.{relation_id}.{group_mapping}.secret" - This function can be used to check if the Provider answered with data in the charm code - when outside an event callback. + def _generate_secret_field_name(self, group_mapping: SecretGroup) -> str: + """Generate unique group_mappings for secrets within a relation context.""" + return f"{PROV_SECRET_PREFIX}{group_mapping}" - Args: - relation_id (int, optional): When provided the check is done only for the relation id - provided, otherwise the check is done for all relations + def _relation_from_secret_label(self, secret_label: str) -> Optional[Relation]: + """Retrieve the relation that belongs to a secret label.""" + contents = secret_label.split(".") - Returns: - True or False + if not (contents and len(contents) >= 3): + return - Raises: - IndexError: If relation_id is provided but that relation does not exist - """ - if relation_id is not None: - try: - relation = [relation for relation in self.relations if relation.id == relation_id][ - 0 - ] - return self._is_resource_created_for_relation(relation) - except IndexError: - raise IndexError(f"relation id {relation_id} cannot be accessed") - else: - return ( - all( - self._is_resource_created_for_relation(relation) for relation in self.relations - ) - if self.relations - else False - ) + contents.pop() # ".secret" at the end + contents.pop() # Group mapping + relation_id = contents.pop() + try: + relation_id = int(relation_id) + except ValueError: + return + # In case '.' character appeared in relation name + relation_name = ".".join(contents) -# General events + try: + return self.get_relation(relation_name, relation_id) + except ModelError: + return + def _group_secret_fields(self, secret_fields: List[str]) -> Dict[SecretGroup, List[str]]: + """Helper function to arrange secret mappings under their group. -class ExtraRoleEvent(RelationEvent): - """Base class for data events.""" + NOTE: All unrecognized items end up in the 'extra' secret bucket. + Make sure only secret fields are passed! + """ + secret_fieldnames_grouped = {} + for key in secret_fields: + if group := self.secret_label_map.get(key): + secret_fieldnames_grouped.setdefault(group, []).append(key) + else: + secret_fieldnames_grouped.setdefault(SECRET_GROUPS.EXTRA, []).append(key) + return secret_fieldnames_grouped + + def _get_group_secret_contents( + self, + relation: Relation, + group: SecretGroup, + secret_fields: Union[Set[str], List[str]] = [], + ) -> Dict[str, str]: + """Helper function to retrieve collective, requested contents of a secret.""" + if (secret := self._get_relation_secret(relation.id, group)) and ( + secret_data := secret.get_content() + ): + return { + k: v for k, v in secret_data.items() if not secret_fields or k in secret_fields + } + return {} + + def _content_for_secret_group( + self, content: Dict[str, str], secret_fields: Set[str], group_mapping: SecretGroup + ) -> Dict[str, str]: + """Select : pairs from input, that belong to this particular Secret group.""" + if group_mapping == SECRET_GROUPS.EXTRA: + return { + k: v + for k, v in content.items() + if k in secret_fields and k not in self.secret_label_map.keys() + } - @property - def extra_user_roles(self) -> Optional[str]: - """Returns the extra user roles that were requested.""" - return self.relation.data[self.relation.app].get("extra-user-roles") + return { + k: v + for k, v in content.items() + if k in secret_fields and self.secret_label_map.get(k) == group_mapping + } + @juju_secrets_only + def _get_relation_secret_data( + self, relation_id: int, group_mapping: SecretGroup, relation_name: Optional[str] = None + ) -> Optional[Dict[str, str]]: + """Retrieve contents of a Juju Secret that's been stored in the relation databag.""" + secret = self._get_relation_secret(relation_id, group_mapping, relation_name) + if secret: + return secret.get_content() -class AuthenticationEvent(RelationEvent): - """Base class for authentication fields for events.""" + # Core operations on Relation Fields manipulations (regardless whether the field is in the databag or in a secret) + # Internal functions to be called directly from transparent public interface functions (+closely related helpers) - @property - def username(self) -> Optional[str]: + def _process_secret_fields( + self, + relation: Relation, + req_secret_fields: Optional[List[str]], + impacted_rel_fields: List[str], + operation: Callable, + *args, + **kwargs, + ) -> Tuple[Dict[str, str], Set[str]]: + """Isolate target secret fields of manipulation, and execute requested operation by Secret Group.""" + result = {} + + # If the relation started on a databag, we just stay on the databag + # (Rolling upgrades may result in a relation starting on databag, getting secrets enabled on-the-fly) + # self.local_app is sufficient to check (ignored if Requires, never has secrets -- works if Provider) + fallback_to_databag = ( + req_secret_fields + and (self.local_unit == self._model.unit and self.local_unit.is_leader()) + and set(req_secret_fields) & set(relation.data[self.component]) + ) + + normal_fields = set(impacted_rel_fields) + if req_secret_fields and self.secrets_enabled and not fallback_to_databag: + normal_fields = normal_fields - set(req_secret_fields) + secret_fields = set(impacted_rel_fields) - set(normal_fields) + + secret_fieldnames_grouped = self._group_secret_fields(list(secret_fields)) + + for group in secret_fieldnames_grouped: + # operation() should return nothing when all goes well + if group_result := operation(relation, group, secret_fields, *args, **kwargs): + # If "meaningful" data was returned, we take it. (Some 'operation'-s only return success/failure.) + if isinstance(group_result, dict): + result.update(group_result) + else: + # If it wasn't found as a secret, let's give it a 2nd chance as "normal" field + # Needed when Juju3 Requires meets Juju2 Provider + normal_fields |= set(secret_fieldnames_grouped[group]) + return (result, normal_fields) + + def _fetch_relation_data_without_secrets( + self, component: Union[Application, Unit], relation: Relation, fields: Optional[List[str]] + ) -> Dict[str, str]: + """Fetching databag contents when no secrets are involved. + + Since the Provider's databag is the only one holding secrest, we can apply + a simplified workflow to read the Require's side's databag. + This is used typically when the Provider side wants to read the Requires side's data, + or when the Requires side may want to read its own data. + """ + if component not in relation.data or not relation.data[component]: + return {} + + if fields: + return { + k: relation.data[component][k] for k in fields if k in relation.data[component] + } + else: + return dict(relation.data[component]) + + def _fetch_relation_data_with_secrets( + self, + component: Union[Application, Unit], + req_secret_fields: Optional[List[str]], + relation: Relation, + fields: Optional[List[str]] = None, + ) -> Dict[str, str]: + """Fetching databag contents when secrets may be involved. + + This function has internal logic to resolve if a requested field may be "hidden" + within a Relation Secret, or directly available as a databag field. Typically + used to read the Provider side's databag (eigher by the Requires side, or by + Provider side itself). + """ + result = {} + normal_fields = [] + + if not fields: + if component not in relation.data: + return {} + + all_fields = list(relation.data[component].keys()) + normal_fields = [field for field in all_fields if not self._is_secret_field(field)] + fields = normal_fields + req_secret_fields if req_secret_fields else normal_fields + + if fields: + result, normal_fields = self._process_secret_fields( + relation, req_secret_fields, fields, self._get_group_secret_contents + ) + + # Processing "normal" fields. May include leftover from what we couldn't retrieve as a secret. + # (Typically when Juju3 Requires meets Juju2 Provider) + if normal_fields: + result.update( + self._fetch_relation_data_without_secrets(component, relation, list(normal_fields)) + ) + return result + + def _update_relation_data_without_secrets( + self, component: Union[Application, Unit], relation: Relation, data: Dict[str, str] + ) -> None: + """Updating databag contents when no secrets are involved.""" + if component not in relation.data or relation.data[component] is None: + return + + if relation: + relation.data[component].update(data) + + def _delete_relation_data_without_secrets( + self, component: Union[Application, Unit], relation: Relation, fields: List[str] + ) -> None: + """Remove databag fields 'fields' from Relation.""" + if component not in relation.data or relation.data[component] is None: + return + + for field in fields: + try: + relation.data[component].pop(field) + except KeyError: + logger.debug( + "Non-existing field '%s' was attempted to be removed from the databag (relation ID: %s)", + str(field), + str(relation.id), + ) + pass + + # Public interface methods + # Handling Relation Fields seamlessly, regardless if in databag or a Juju Secret + + def as_dict(self, relation_id: int) -> UserDict: + """Dict behavior representation of the Abstract Data.""" + return DataDict(self, relation_id) + + def get_relation(self, relation_name, relation_id) -> Relation: + """Safe way of retrieving a relation.""" + relation = self._model.get_relation(relation_name, relation_id) + + if not relation: + raise DataInterfacesError( + "Relation %s %s couldn't be retrieved", relation_name, relation_id + ) + + return relation + + def fetch_relation_data( + self, + relation_ids: Optional[List[int]] = None, + fields: Optional[List[str]] = None, + relation_name: Optional[str] = None, + ) -> Dict[int, Dict[str, str]]: + """Retrieves data from relation. + + This function can be used to retrieve data from a relation + in the charm code when outside an event callback. + Function cannot be used in `*-relation-broken` events and will raise an exception. + + Returns: + a dict of the values stored in the relation data bag + for all relation instances (indexed by the relation ID). + """ + if not relation_name: + relation_name = self.relation_name + + relations = [] + if relation_ids: + relations = [ + self.get_relation(relation_name, relation_id) for relation_id in relation_ids + ] + else: + relations = self.relations + + data = {} + for relation in relations: + if not relation_ids or (relation_ids and relation.id in relation_ids): + data[relation.id] = self._fetch_specific_relation_data(relation, fields) + return data + + def fetch_relation_field( + self, relation_id: int, field: str, relation_name: Optional[str] = None + ) -> Optional[str]: + """Get a single field from the relation data.""" + return ( + self.fetch_relation_data([relation_id], [field], relation_name) + .get(relation_id, {}) + .get(field) + ) + + def fetch_my_relation_data( + self, + relation_ids: Optional[List[int]] = None, + fields: Optional[List[str]] = None, + relation_name: Optional[str] = None, + ) -> Optional[Dict[int, Dict[str, str]]]: + """Fetch data of the 'owner' (or 'this app') side of the relation. + + NOTE: Since only the leader can read the relation's 'this_app'-side + Application databag, the functionality is limited to leaders + """ + if not relation_name: + relation_name = self.relation_name + + relations = [] + if relation_ids: + relations = [ + self.get_relation(relation_name, relation_id) for relation_id in relation_ids + ] + else: + relations = self.relations + + data = {} + for relation in relations: + if not relation_ids or relation.id in relation_ids: + data[relation.id] = self._fetch_my_specific_relation_data(relation, fields) + return data + + def fetch_my_relation_field( + self, relation_id: int, field: str, relation_name: Optional[str] = None + ) -> Optional[str]: + """Get a single field from the relation data -- owner side. + + NOTE: Since only the leader can read the relation's 'this_app'-side + Application databag, the functionality is limited to leaders + """ + if relation_data := self.fetch_my_relation_data([relation_id], [field], relation_name): + return relation_data.get(relation_id, {}).get(field) + + @leader_only + def update_relation_data(self, relation_id: int, data: dict) -> None: + """Update the data within the relation.""" + relation_name = self.relation_name + relation = self.get_relation(relation_name, relation_id) + return self._update_relation_data(relation, data) + + @leader_only + def delete_relation_data(self, relation_id: int, fields: List[str]) -> None: + """Remove field from the relation.""" + relation_name = self.relation_name + relation = self.get_relation(relation_name, relation_id) + return self._delete_relation_data(relation, fields) + + +class EventHandlers(Object): + """Requires-side of the relation.""" + + def __init__(self, charm: CharmBase, relation_data: Data, unique_key: str = ""): + """Manager of base client relations.""" + if not unique_key: + unique_key = relation_data.relation_name + super().__init__(charm, unique_key) + + self.charm = charm + self.relation_data = relation_data + + self.framework.observe( + charm.on[self.relation_data.relation_name].relation_changed, + self._on_relation_changed_event, + ) + + def _diff(self, event: RelationChangedEvent) -> Diff: + """Retrieves the diff of the data in the relation changed databag. + + Args: + event: relation changed event. + + Returns: + a Diff instance containing the added, deleted and changed + keys from the event relation databag. + """ + return diff(event, self.relation_data.data_component) + + @abstractmethod + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation data has changed.""" + raise NotImplementedError + + +# Base ProviderData and RequiresData + + +class ProviderData(Data): + """Base provides-side of the data products relation.""" + + def __init__( + self, + model: Model, + relation_name: str, + ) -> None: + super().__init__(model, relation_name) + self.data_component = self.local_app + + # Private methods handling secrets + + @juju_secrets_only + def _add_relation_secret( + self, + relation: Relation, + group_mapping: SecretGroup, + secret_fields: Set[str], + data: Dict[str, str], + uri_to_databag=True, + ) -> bool: + """Add a new Juju Secret that will be registered in the relation databag.""" + secret_field = self._generate_secret_field_name(group_mapping) + if uri_to_databag and relation.data[self.component].get(secret_field): + logging.error("Secret for relation %s already exists, not adding again", relation.id) + return False + + content = self._content_for_secret_group(data, secret_fields, group_mapping) + + label = self._generate_secret_label(self.relation_name, relation.id, group_mapping) + secret = self.secrets.add(label, content, relation) + + # According to lint we may not have a Secret ID + if uri_to_databag and secret.meta and secret.meta.id: + relation.data[self.component][secret_field] = secret.meta.id + + # Return the content that was added + return True + + @juju_secrets_only + def _update_relation_secret( + self, + relation: Relation, + group_mapping: SecretGroup, + secret_fields: Set[str], + data: Dict[str, str], + ) -> bool: + """Update the contents of an existing Juju Secret, referred in the relation databag.""" + secret = self._get_relation_secret(relation.id, group_mapping) + + if not secret: + logging.error("Can't update secret for relation %s", relation.id) + return False + + content = self._content_for_secret_group(data, secret_fields, group_mapping) + + old_content = secret.get_content() + full_content = copy.deepcopy(old_content) + full_content.update(content) + secret.set_content(full_content) + + # Return True on success + return True + + def _add_or_update_relation_secrets( + self, + relation: Relation, + group: SecretGroup, + secret_fields: Set[str], + data: Dict[str, str], + uri_to_databag=True, + ) -> bool: + """Update contents for Secret group. If the Secret doesn't exist, create it.""" + if self._get_relation_secret(relation.id, group): + return self._update_relation_secret(relation, group, secret_fields, data) + else: + return self._add_relation_secret(relation, group, secret_fields, data, uri_to_databag) + + @juju_secrets_only + def _delete_relation_secret( + self, relation: Relation, group: SecretGroup, secret_fields: List[str], fields: List[str] + ) -> bool: + """Update the contents of an existing Juju Secret, referred in the relation databag.""" + secret = self._get_relation_secret(relation.id, group) + + if not secret: + logging.error("Can't delete secret for relation %s", str(relation.id)) + return False + + old_content = secret.get_content() + new_content = copy.deepcopy(old_content) + for field in fields: + try: + new_content.pop(field) + except KeyError: + logging.debug( + "Non-existing secret was attempted to be removed %s, %s", + str(relation.id), + str(field), + ) + return False + + # Remove secret from the relation if it's fully gone + if not new_content: + field = self._generate_secret_field_name(group) + try: + relation.data[self.component].pop(field) + except KeyError: + pass + label = self._generate_secret_label(self.relation_name, relation.id, group) + self.secrets.remove(label) + else: + secret.set_content(new_content) + + # Return the content that was removed + return True + + # Mandatory internal overrides + + @juju_secrets_only + def _get_relation_secret( + self, relation_id: int, group_mapping: SecretGroup, relation_name: Optional[str] = None + ) -> Optional[CachedSecret]: + """Retrieve a Juju Secret that's been stored in the relation databag.""" + if not relation_name: + relation_name = self.relation_name + + label = self._generate_secret_label(relation_name, relation_id, group_mapping) + if secret := self.secrets.get(label): + return secret + + relation = self._model.get_relation(relation_name, relation_id) + if not relation: + return + + secret_field = self._generate_secret_field_name(group_mapping) + if secret_uri := relation.data[self.local_app].get(secret_field): + return self.secrets.get(label, secret_uri) + + def _fetch_specific_relation_data( + self, relation: Relation, fields: Optional[List[str]] + ) -> Dict[str, str]: + """Fetching relation data for Provider. + + NOTE: Since all secret fields are in the Provider side of the databag, we don't need to worry about that + """ + if not relation.app: + return {} + + return self._fetch_relation_data_without_secrets(relation.app, relation, fields) + + def _fetch_my_specific_relation_data( + self, relation: Relation, fields: Optional[List[str]] + ) -> dict: + """Fetching our own relation data.""" + secret_fields = None + if relation.app: + secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) + + return self._fetch_relation_data_with_secrets( + self.local_app, + secret_fields, + relation, + fields, + ) + + def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: + """Set values for fields not caring whether it's a secret or not.""" + req_secret_fields = [] + if relation.app: + req_secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) + + _, normal_fields = self._process_secret_fields( + relation, + req_secret_fields, + list(data), + self._add_or_update_relation_secrets, + data=data, + ) + + normal_content = {k: v for k, v in data.items() if k in normal_fields} + self._update_relation_data_without_secrets(self.local_app, relation, normal_content) + + def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: + """Delete fields from the Relation not caring whether it's a secret or not.""" + req_secret_fields = [] + if relation.app: + req_secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) + + _, normal_fields = self._process_secret_fields( + relation, req_secret_fields, fields, self._delete_relation_secret, fields=fields + ) + self._delete_relation_data_without_secrets(self.local_app, relation, list(normal_fields)) + + # Public methods - "native" + + def set_credentials(self, relation_id: int, username: str, password: str) -> None: + """Set credentials. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + username: user that was created. + password: password of the created user. + """ + self.update_relation_data(relation_id, {"username": username, "password": password}) + + def set_tls(self, relation_id: int, tls: str) -> None: + """Set whether TLS is enabled. + + Args: + relation_id: the identifier for a particular relation. + tls: whether tls is enabled (True or False). + """ + self.update_relation_data(relation_id, {"tls": tls}) + + def set_tls_ca(self, relation_id: int, tls_ca: str) -> None: + """Set the TLS CA in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + tls_ca: TLS certification authority. + """ + self.update_relation_data(relation_id, {"tls-ca": tls_ca}) + + # Public functions -- inherited + + fetch_my_relation_data = leader_only(Data.fetch_my_relation_data) + fetch_my_relation_field = leader_only(Data.fetch_my_relation_field) + + +class RequirerData(Data): + """Requirer-side of the relation.""" + + SECRET_FIELDS = ["username", "password", "tls", "tls-ca", "uris"] + + def __init__( + self, + model, + relation_name: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + ): + """Manager of base client relations.""" + super().__init__(model, relation_name) + self.extra_user_roles = extra_user_roles + self._secret_fields = list(self.SECRET_FIELDS) + if additional_secret_fields: + self._secret_fields += additional_secret_fields + self.data_component = self.local_unit + + @property + def secret_fields(self) -> Optional[List[str]]: + """Local access to secrets field, in case they are being used.""" + if self.secrets_enabled: + return self._secret_fields + + # Internal helper functions + + def _register_secret_to_relation( + self, relation_name: str, relation_id: int, secret_id: str, group: SecretGroup + ): + """Fetch secrets and apply local label on them. + + [MAGIC HERE] + If we fetch a secret using get_secret(id=, label=), + then will be "stuck" on the Secret object, whenever it may + appear (i.e. as an event attribute, or fetched manually) on future occasions. + + This will allow us to uniquely identify the secret on Provider side (typically on + 'secret-changed' events), and map it to the corresponding relation. + """ + label = self._generate_secret_label(relation_name, relation_id, group) + + # Fetchin the Secret's meta information ensuring that it's locally getting registered with + CachedSecret(self._model, self.component, label, secret_id).meta + + def _register_secrets_to_relation(self, relation: Relation, params_name_list: List[str]): + """Make sure that secrets of the provided list are locally 'registered' from the databag. + + More on 'locally registered' magic is described in _register_secret_to_relation() method + """ + if not relation.app: + return + + for group in SECRET_GROUPS.groups(): + secret_field = self._generate_secret_field_name(group) + if secret_field in params_name_list: + if secret_uri := relation.data[relation.app].get(secret_field): + self._register_secret_to_relation( + relation.name, relation.id, secret_uri, group + ) + + def _is_resource_created_for_relation(self, relation: Relation) -> bool: + if not relation.app: + return False + + data = self.fetch_relation_data([relation.id], ["username", "password"]).get( + relation.id, {} + ) + return bool(data.get("username")) and bool(data.get("password")) + + def is_resource_created(self, relation_id: Optional[int] = None) -> bool: + """Check if the resource has been created. + + This function can be used to check if the Provider answered with data in the charm code + when outside an event callback. + + Args: + relation_id (int, optional): When provided the check is done only for the relation id + provided, otherwise the check is done for all relations + + Returns: + True or False + + Raises: + IndexError: If relation_id is provided but that relation does not exist + """ + if relation_id is not None: + try: + relation = [relation for relation in self.relations if relation.id == relation_id][ + 0 + ] + return self._is_resource_created_for_relation(relation) + except IndexError: + raise IndexError(f"relation id {relation_id} cannot be accessed") + else: + return ( + all( + self._is_resource_created_for_relation(relation) for relation in self.relations + ) + if self.relations + else False + ) + + # Mandatory internal overrides + + @juju_secrets_only + def _get_relation_secret( + self, relation_id: int, group: SecretGroup, relation_name: Optional[str] = None + ) -> Optional[CachedSecret]: + """Retrieve a Juju Secret that's been stored in the relation databag.""" + if not relation_name: + relation_name = self.relation_name + + label = self._generate_secret_label(relation_name, relation_id, group) + return self.secrets.get(label) + + def _fetch_specific_relation_data( + self, relation, fields: Optional[List[str]] = None + ) -> Dict[str, str]: + """Fetching Requirer data -- that may include secrets.""" + if not relation.app: + return {} + return self._fetch_relation_data_with_secrets( + relation.app, self.secret_fields, relation, fields + ) + + def _fetch_my_specific_relation_data(self, relation, fields: Optional[List[str]]) -> dict: + """Fetching our own relation data.""" + return self._fetch_relation_data_without_secrets(self.local_app, relation, fields) + + def _update_relation_data(self, relation: Relation, data: dict) -> None: + """Updates a set of key-value pairs in the relation. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation: the particular relation. + data: dict containing the key-value pairs + that should be updated in the relation. + """ + return self._update_relation_data_without_secrets(self.local_app, relation, data) + + def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: + """Deletes a set of fields from the relation. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation: the particular relation. + fields: list containing the field names that should be removed from the relation. + """ + return self._delete_relation_data_without_secrets(self.local_app, relation, fields) + + # Public functions -- inherited + + fetch_my_relation_data = leader_only(Data.fetch_my_relation_data) + fetch_my_relation_field = leader_only(Data.fetch_my_relation_field) + + +class RequirerEventHandlers(EventHandlers): + """Requires-side of the relation.""" + + def __init__(self, charm: CharmBase, relation_data: RequirerData, unique_key: str = ""): + """Manager of base client relations.""" + super().__init__(charm, relation_data, unique_key) + + self.framework.observe( + self.charm.on[relation_data.relation_name].relation_created, + self._on_relation_created_event, + ) + self.framework.observe( + charm.on.secret_changed, + self._on_secret_changed_event, + ) + + # Event handlers + + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the relation is created.""" + if not self.relation_data.local_unit.is_leader(): + return + + if self.relation_data.secret_fields: # pyright: ignore [reportAttributeAccessIssue] + set_encoded_field( + event.relation, + self.relation_data.component, + REQ_SECRET_FIELDS, + self.relation_data.secret_fields, # pyright: ignore [reportAttributeAccessIssue] + ) + + @abstractmethod + def _on_secret_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation data has changed.""" + raise NotImplementedError + + +################################################################################ +# Peer Relation Data +################################################################################ + + +class DataPeerData(RequirerData, ProviderData): + """Represents peer relations data.""" + + SECRET_FIELDS = [] + SECRET_FIELD_NAME = "internal_secret" + SECRET_LABEL_MAP = {} + + def __init__( + self, + model, + relation_name: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + additional_secret_group_mapping: Dict[str, str] = {}, + secret_field_name: Optional[str] = None, + deleted_label: Optional[str] = None, + ): + """Manager of base client relations.""" + RequirerData.__init__( + self, + model, + relation_name, + extra_user_roles, + additional_secret_fields, + ) + self.secret_field_name = secret_field_name if secret_field_name else self.SECRET_FIELD_NAME + self.deleted_label = deleted_label + self._secret_label_map = {} + # Secrets that are being dynamically added within the scope of this event handler run + self._new_secrets = [] + self._additional_secret_group_mapping = additional_secret_group_mapping + + for group, fields in additional_secret_group_mapping.items(): + if group not in SECRET_GROUPS.groups(): + setattr(SECRET_GROUPS, group, group) + for field in fields: + secret_group = SECRET_GROUPS.get_group(group) + internal_field = self._field_to_internal_name(field, secret_group) + self._secret_label_map.setdefault(group, []).append(internal_field) + self._secret_fields.append(internal_field) + + @property + def scope(self) -> Optional[Scope]: + """Turn component information into Scope.""" + if isinstance(self.component, Application): + return Scope.APP + if isinstance(self.component, Unit): + return Scope.UNIT + + @property + def secret_label_map(self) -> Dict[str, str]: + """Property storing secret mappings.""" + return self._secret_label_map + + @property + def static_secret_fields(self) -> List[str]: + """Re-definition of the property in a way that dynamically extended list is retrieved.""" + return self._secret_fields + + @property + def secret_fields(self) -> List[str]: + """Re-definition of the property in a way that dynamically extended list is retrieved.""" + return ( + self.static_secret_fields if self.static_secret_fields else self.current_secret_fields + ) + + @property + def current_secret_fields(self) -> List[str]: + """Helper method to get all currently existing secret fields (added statically or dynamically).""" + if not self.secrets_enabled: + return [] + + if len(self._model.relations[self.relation_name]) > 1: + raise ValueError(f"More than one peer relation on {self.relation_name}") + + relation = self._model.relations[self.relation_name][0] + fields = [] + + ignores = [SECRET_GROUPS.get_group("user"), SECRET_GROUPS.get_group("tls")] + for group in SECRET_GROUPS.groups(): + if group in ignores: + continue + if content := self._get_group_secret_contents(relation, group): + fields += list(content.keys()) + return list(set(fields) | set(self._new_secrets)) + + @dynamic_secrets_only + def set_secret( + self, + relation_id: int, + field: str, + value: str, + group_mapping: Optional[SecretGroup] = None, + ) -> None: + """Public interface method to add a Relation Data field specifically as a Juju Secret. + + Args: + relation_id: ID of the relation + field: The secret field that is to be added + value: The string value of the secret + group_mapping: The name of the "secret group", in case the field is to be added to an existing secret + """ + full_field = self._field_to_internal_name(field, group_mapping) + if self.secrets_enabled and full_field not in self.current_secret_fields: + self._new_secrets.append(full_field) + if self._no_group_with_databag(field, full_field): + self.update_relation_data(relation_id, {full_field: value}) + + # Unlike for set_secret(), there's no harm using this operation with static secrets + # The restricion is only added to keep the concept clear + @dynamic_secrets_only + def get_secret( + self, + relation_id: int, + field: str, + group_mapping: Optional[SecretGroup] = None, + ) -> Optional[str]: + """Public interface method to fetch secrets only.""" + full_field = self._field_to_internal_name(field, group_mapping) + if ( + self.secrets_enabled + and full_field not in self.current_secret_fields + and field not in self.current_secret_fields + ): + return + if self._no_group_with_databag(field, full_field): + return self.fetch_my_relation_field(relation_id, full_field) + + @dynamic_secrets_only + def delete_secret( + self, + relation_id: int, + field: str, + group_mapping: Optional[SecretGroup] = None, + ) -> Optional[str]: + """Public interface method to delete secrets only.""" + full_field = self._field_to_internal_name(field, group_mapping) + if self.secrets_enabled and full_field not in self.current_secret_fields: + logger.warning(f"Secret {field} from group {group_mapping} was not found") + return + if self._no_group_with_databag(field, full_field): + self.delete_relation_data(relation_id, [full_field]) + + # Helpers + + @staticmethod + def _field_to_internal_name(field: str, group: Optional[SecretGroup]) -> str: + if not group or group == SECRET_GROUPS.EXTRA: + return field + return f"{field}{GROUP_SEPARATOR}{group}" + + @staticmethod + def _internal_name_to_field(name: str) -> Tuple[str, SecretGroup]: + parts = name.split(GROUP_SEPARATOR) + if not len(parts) > 1: + return (parts[0], SECRET_GROUPS.EXTRA) + secret_group = SECRET_GROUPS.get_group(parts[1]) + if not secret_group: + raise ValueError(f"Invalid secret field {name}") + return (parts[0], secret_group) + + def _group_secret_fields(self, secret_fields: List[str]) -> Dict[SecretGroup, List[str]]: + """Helper function to arrange secret mappings under their group. + + NOTE: All unrecognized items end up in the 'extra' secret bucket. + Make sure only secret fields are passed! + """ + secret_fieldnames_grouped = {} + for key in secret_fields: + field, group = self._internal_name_to_field(key) + secret_fieldnames_grouped.setdefault(group, []).append(field) + return secret_fieldnames_grouped + + def _content_for_secret_group( + self, content: Dict[str, str], secret_fields: Set[str], group_mapping: SecretGroup + ) -> Dict[str, str]: + """Select : pairs from input, that belong to this particular Secret group.""" + if group_mapping == SECRET_GROUPS.EXTRA: + return {k: v for k, v in content.items() if k in self.secret_fields} + return { + self._internal_name_to_field(k)[0]: v + for k, v in content.items() + if k in self.secret_fields + } + + # Backwards compatibility + + def _check_deleted_label(self, relation, fields) -> None: + """Helper function for legacy behavior.""" + current_data = self.fetch_my_relation_data([relation.id], fields) + if current_data is not None: + # Check if the secret we wanna delete actually exists + # Given the "deleted label", here we can't rely on the default mechanism (i.e. 'key not found') + if non_existent := (set(fields) & set(self.secret_fields)) - set( + current_data.get(relation.id, []) + ): + logger.debug( + "Non-existing secret %s was attempted to be removed.", + ", ".join(non_existent), + ) + + def _remove_secret_from_databag(self, relation, fields: List[str]) -> None: + """For Rolling Upgrades -- when moving from databag to secrets usage. + + Practically what happens here is to remove stuff from the databag that is + to be stored in secrets. + """ + if not self.secret_fields: + return + + secret_fields_passed = set(self.secret_fields) & set(fields) + for field in secret_fields_passed: + if self._fetch_relation_data_without_secrets(self.component, relation, [field]): + self._delete_relation_data_without_secrets(self.component, relation, [field]) + + def _remove_secret_field_name_from_databag(self, relation) -> None: + """Making sure that the old databag URI is gone. + + This action should not be executed more than once. + """ + # Nothing to do if 'internal-secret' is not in the databag + if not (relation.data[self.component].get(self._generate_secret_field_name())): + return + + # Making sure that the secret receives its label + # (This should have happened by the time we get here, rather an extra security measure.) + secret = self._get_relation_secret(relation.id) + + # Either app scope secret with leader executing, or unit scope secret + leader_or_unit_scope = self.component != self.local_app or self.local_unit.is_leader() + if secret and leader_or_unit_scope: + # Databag reference to the secret URI can be removed, now that it's labelled + relation.data[self.component].pop(self._generate_secret_field_name(), None) + + def _previous_labels(self) -> List[str]: + """Generator for legacy secret label names, for backwards compatibility.""" + result = [] + members = [self._model.app.name] + if self.scope: + members.append(self.scope.value) + result.append(f"{'.'.join(members)}") + return result + + def _no_group_with_databag(self, field: str, full_field: str) -> bool: + """Check that no secret group is attempted to be used together with databag.""" + if not self.secrets_enabled and full_field != field: + logger.error( + f"Can't access {full_field}: no secrets available (i.e. no secret groups either)." + ) + return False + return True + + # Event handlers + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + pass + + def _on_secret_changed_event(self, event: SecretChangedEvent) -> None: + """Event emitted when the secret has changed.""" + pass + + # Overrides of Relation Data handling functions + + def _generate_secret_label( + self, relation_name: str, relation_id: int, group_mapping: SecretGroup + ) -> str: + members = [relation_name, self._model.app.name] + if self.scope: + members.append(self.scope.value) + if group_mapping != SECRET_GROUPS.EXTRA: + members.append(group_mapping) + return f"{'.'.join(members)}" + + def _generate_secret_field_name(self, group_mapping: SecretGroup = SECRET_GROUPS.EXTRA) -> str: + """Generate unique group_mappings for secrets within a relation context.""" + return f"{self.secret_field_name}" + + @juju_secrets_only + def _get_relation_secret( + self, + relation_id: int, + group_mapping: SecretGroup = SECRET_GROUPS.EXTRA, + relation_name: Optional[str] = None, + ) -> Optional[CachedSecret]: + """Retrieve a Juju Secret specifically for peer relations. + + In case this code may be executed within a rolling upgrade, and we may need to + migrate secrets from the databag to labels, we make sure to stick the correct + label on the secret, and clean up the local databag. + """ + if not relation_name: + relation_name = self.relation_name + + relation = self._model.get_relation(relation_name, relation_id) + if not relation: + return + + label = self._generate_secret_label(relation_name, relation_id, group_mapping) + secret_uri = relation.data[self.component].get(self._generate_secret_field_name(), None) + + # URI or legacy label is only to applied when moving single legacy secret to a (new) label + if group_mapping == SECRET_GROUPS.EXTRA: + # Fetching the secret with fallback to URI (in case label is not yet known) + # Label would we "stuck" on the secret in case it is found + return self.secrets.get(label, secret_uri, legacy_labels=self._previous_labels()) + return self.secrets.get(label) + + def _get_group_secret_contents( + self, + relation: Relation, + group: SecretGroup, + secret_fields: Union[Set[str], List[str]] = [], + ) -> Dict[str, str]: + """Helper function to retrieve collective, requested contents of a secret.""" + secret_fields = [self._internal_name_to_field(k)[0] for k in secret_fields] + result = super()._get_group_secret_contents(relation, group, secret_fields) + if self.deleted_label: + result = {key: result[key] for key in result if result[key] != self.deleted_label} + if self._additional_secret_group_mapping: + return {self._field_to_internal_name(key, group): result[key] for key in result} + return result + + @either_static_or_dynamic_secrets + def _fetch_my_specific_relation_data( + self, relation: Relation, fields: Optional[List[str]] + ) -> Dict[str, str]: + """Fetch data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + return self._fetch_relation_data_with_secrets( + self.component, self.secret_fields, relation, fields + ) + + @either_static_or_dynamic_secrets + def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: + """Update data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + self._remove_secret_from_databag(relation, list(data.keys())) + _, normal_fields = self._process_secret_fields( + relation, + self.secret_fields, + list(data), + self._add_or_update_relation_secrets, + data=data, + uri_to_databag=False, + ) + self._remove_secret_field_name_from_databag(relation) + + normal_content = {k: v for k, v in data.items() if k in normal_fields} + self._update_relation_data_without_secrets(self.component, relation, normal_content) + + @either_static_or_dynamic_secrets + def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: + """Delete data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + if self.secret_fields and self.deleted_label: + # Legacy, backwards compatibility + self._check_deleted_label(relation, fields) + + _, normal_fields = self._process_secret_fields( + relation, + self.secret_fields, + fields, + self._update_relation_secret, + data={field: self.deleted_label for field in fields}, + ) + else: + _, normal_fields = self._process_secret_fields( + relation, self.secret_fields, fields, self._delete_relation_secret, fields=fields + ) + self._delete_relation_data_without_secrets(self.component, relation, list(normal_fields)) + + def fetch_relation_data( + self, + relation_ids: Optional[List[int]] = None, + fields: Optional[List[str]] = None, + relation_name: Optional[str] = None, + ) -> Dict[int, Dict[str, str]]: + """This method makes no sense for a Peer Relation.""" + raise NotImplementedError( + "Peer Relation only supports 'self-side' fetch methods: " + "fetch_my_relation_data() and fetch_my_relation_field()" + ) + + def fetch_relation_field( + self, relation_id: int, field: str, relation_name: Optional[str] = None + ) -> Optional[str]: + """This method makes no sense for a Peer Relation.""" + raise NotImplementedError( + "Peer Relation only supports 'self-side' fetch methods: " + "fetch_my_relation_data() and fetch_my_relation_field()" + ) + + # Public functions -- inherited + + fetch_my_relation_data = Data.fetch_my_relation_data + fetch_my_relation_field = Data.fetch_my_relation_field + + +class DataPeerEventHandlers(RequirerEventHandlers): + """Requires-side of the relation.""" + + def __init__(self, charm: CharmBase, relation_data: RequirerData, unique_key: str = ""): + """Manager of base client relations.""" + super().__init__(charm, relation_data, unique_key) + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + pass + + def _on_secret_changed_event(self, event: SecretChangedEvent) -> None: + """Event emitted when the secret has changed.""" + pass + + +class DataPeer(DataPeerData, DataPeerEventHandlers): + """Represents peer relations.""" + + def __init__( + self, + charm, + relation_name: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + additional_secret_group_mapping: Dict[str, str] = {}, + secret_field_name: Optional[str] = None, + deleted_label: Optional[str] = None, + unique_key: str = "", + ): + DataPeerData.__init__( + self, + charm.model, + relation_name, + extra_user_roles, + additional_secret_fields, + additional_secret_group_mapping, + secret_field_name, + deleted_label, + ) + DataPeerEventHandlers.__init__(self, charm, self, unique_key) + + +class DataPeerUnitData(DataPeerData): + """Unit data abstraction representation.""" + + SCOPE = Scope.UNIT + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class DataPeerUnit(DataPeerUnitData, DataPeerEventHandlers): + """Unit databag representation.""" + + def __init__( + self, + charm, + relation_name: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + additional_secret_group_mapping: Dict[str, str] = {}, + secret_field_name: Optional[str] = None, + deleted_label: Optional[str] = None, + unique_key: str = "", + ): + DataPeerData.__init__( + self, + charm.model, + relation_name, + extra_user_roles, + additional_secret_fields, + additional_secret_group_mapping, + secret_field_name, + deleted_label, + ) + DataPeerEventHandlers.__init__(self, charm, self, unique_key) + + +class DataPeerOtherUnitData(DataPeerUnitData): + """Unit data abstraction representation.""" + + def __init__(self, unit: Unit, *args, **kwargs): + super().__init__(*args, **kwargs) + self.local_unit = unit + self.component = unit + + def update_relation_data(self, relation_id: int, data: dict) -> None: + """This method makes no sense for a Other Peer Relation.""" + raise NotImplementedError("It's not possible to update data of another unit.") + + def delete_relation_data(self, relation_id: int, fields: List[str]) -> None: + """This method makes no sense for a Other Peer Relation.""" + raise NotImplementedError("It's not possible to delete data of another unit.") + + +class DataPeerOtherUnitEventHandlers(DataPeerEventHandlers): + """Requires-side of the relation.""" + + def __init__(self, charm: CharmBase, relation_data: DataPeerUnitData): + """Manager of base client relations.""" + unique_key = f"{relation_data.relation_name}-{relation_data.local_unit.name}" + super().__init__(charm, relation_data, unique_key=unique_key) + + +class DataPeerOtherUnit(DataPeerOtherUnitData, DataPeerOtherUnitEventHandlers): + """Unit databag representation for another unit than the executor.""" + + def __init__( + self, + unit: Unit, + charm: CharmBase, + relation_name: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + additional_secret_group_mapping: Dict[str, str] = {}, + secret_field_name: Optional[str] = None, + deleted_label: Optional[str] = None, + ): + DataPeerOtherUnitData.__init__( + self, + unit, + charm.model, + relation_name, + extra_user_roles, + additional_secret_fields, + additional_secret_group_mapping, + secret_field_name, + deleted_label, + ) + DataPeerOtherUnitEventHandlers.__init__(self, charm, self) + + +################################################################################ +# Cross-charm Relatoins Data Handling and Evenets +################################################################################ + +# Generic events + + +class ExtraRoleEvent(RelationEvent): + """Base class for data events.""" + + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-user-roles") + + +class RelationEventWithSecret(RelationEvent): + """Base class for Relation Events that need to handle secrets.""" + + @property + def _secrets(self) -> dict: + """Caching secrets to avoid fetching them each time a field is referrd. + + DON'T USE the encapsulated helper variable outside of this function + """ + if not hasattr(self, "_cached_secrets"): + self._cached_secrets = {} + return self._cached_secrets + + def _get_secret(self, group) -> Optional[Dict[str, str]]: + """Retrieveing secrets.""" + if not self.app: + return + if not self._secrets.get(group): + self._secrets[group] = None + secret_field = f"{PROV_SECRET_PREFIX}{group}" + if secret_uri := self.relation.data[self.app].get(secret_field): + secret = self.framework.model.get_secret(id=secret_uri) + self._secrets[group] = secret.get_content() + return self._secrets[group] + + @property + def secrets_enabled(self): + """Is this Juju version allowing for Secrets usage?""" + return JujuVersion.from_environ().has_secrets + + +class AuthenticationEvent(RelationEventWithSecret): + """Base class for authentication fields for events. + + The amount of logic added here is not ideal -- but this was the only way to preserve + the interface when moving to Juju Secrets + """ + + @property + def username(self) -> Optional[str]: """Returns the created username.""" + if not self.relation.app: + return None + + if self.secrets_enabled: + secret = self._get_secret("user") + if secret: + return secret.get("username") + return self.relation.data[self.relation.app].get("username") @property def password(self) -> Optional[str]: """Returns the password for the created user.""" + if not self.relation.app: + return None + + if self.secrets_enabled: + secret = self._get_secret("user") + if secret: + return secret.get("password") + return self.relation.data[self.relation.app].get("password") @property def tls(self) -> Optional[str]: """Returns whether TLS is configured.""" + if not self.relation.app: + return None + + if self.secrets_enabled: + secret = self._get_secret("tls") + if secret: + return secret.get("tls") + return self.relation.data[self.relation.app].get("tls") @property def tls_ca(self) -> Optional[str]: """Returns TLS CA.""" + if not self.relation.app: + return None + + if self.secrets_enabled: + secret = self._get_secret("tls") + if secret: + return secret.get("tls-ca") + return self.relation.data[self.relation.app].get("tls-ca") @@ -638,12 +2395,26 @@ class DatabaseProvidesEvent(RelationEvent): @property def database(self) -> Optional[str]: """Returns the database that was requested.""" + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("database") class DatabaseRequestedEvent(DatabaseProvidesEvent, ExtraRoleEvent): """Event emitted when a new database is requested for use on this relation.""" + @property + def external_node_connectivity(self) -> bool: + """Returns the requested external_node_connectivity field.""" + if not self.relation.app: + return False + + return ( + self.relation.data[self.relation.app].get("external-node-connectivity", "false") + == "true" + ) + class DatabaseProvidesEvents(CharmEvents): """Database events. @@ -654,22 +2425,39 @@ class DatabaseProvidesEvents(CharmEvents): database_requested = EventSource(DatabaseRequestedEvent) -class DatabaseRequiresEvent(RelationEvent): +class DatabaseRequiresEvent(RelationEventWithSecret): """Base class for database events.""" @property def database(self) -> Optional[str]: """Returns the database name.""" + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("database") @property def endpoints(self) -> Optional[str]: - """Returns a comma separated list of read/write endpoints.""" + """Returns a comma separated list of read/write endpoints. + + In VM charms, this is the primary's address. + In kubernetes charms, this is the service to the primary pod. + """ + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("endpoints") @property def read_only_endpoints(self) -> Optional[str]: - """Returns a comma separated list of read only endpoints.""" + """Returns a comma separated list of read only endpoints. + + In VM charms, this is the address of all the secondary instances. + In kubernetes charms, this is the service to all replica pod instances. + """ + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("read-only-endpoints") @property @@ -678,6 +2466,9 @@ def replset(self) -> Optional[str]: MongoDB only. """ + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("replset") @property @@ -686,6 +2477,14 @@ def uris(self) -> Optional[str]: MongoDB, Redis, OpenSearch. """ + if not self.relation.app: + return None + + if self.secrets_enabled: + secret = self._get_secret("user") + if secret: + return secret.get("uris") + return self.relation.data[self.relation.app].get("uris") @property @@ -694,6 +2493,9 @@ def version(self) -> Optional[str]: Version as informed by the database daemon. """ + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("version") @@ -723,27 +2525,11 @@ class DatabaseRequiresEvents(CharmEvents): # Database Provider and Requires -class DatabaseProvides(DataProvides): - """Provider-side of the database relations.""" - - on = DatabaseProvidesEvents() - - def __init__(self, charm: CharmBase, relation_name: str) -> None: - super().__init__(charm, relation_name) - - def _on_relation_changed(self, event: RelationChangedEvent) -> None: - """Event emitted when the relation has changed.""" - # Only the leader should handle this event. - if not self.local_unit.is_leader(): - return +class DatabaseProviderData(ProviderData): + """Provider-side data of the database relations.""" - # Check which data has changed to emit customs events. - diff = self._diff(event) - - # Emit a database requested event if the setup key (database name and optional - # extra user roles) was added to the relation databag by the application. - if "database" in diff.added: - self.on.database_requested.emit(event.relation, app=event.app, unit=event.unit) + def __init__(self, model: Model, relation_name: str) -> None: + super().__init__(model, relation_name) def set_database(self, relation_id: int, database_name: str) -> None: """Set database name. @@ -755,7 +2541,7 @@ def set_database(self, relation_id: int, database_name: str) -> None: relation_id: the identifier for a particular relation. database_name: database name. """ - self._update_relation_data(relation_id, {"database": database_name}) + self.update_relation_data(relation_id, {"database": database_name}) def set_endpoints(self, relation_id: int, connection_strings: str) -> None: """Set database primary connections. @@ -763,11 +2549,15 @@ def set_endpoints(self, relation_id: int, connection_strings: str) -> None: This function writes in the application data bag, therefore, only the leader unit can call it. + In VM charms, only the primary's address should be passed as an endpoint. + In kubernetes charms, the service endpoint to the primary pod should be + passed as an endpoint. + Args: relation_id: the identifier for a particular relation. connection_strings: database hosts and ports comma separated list. """ - self._update_relation_data(relation_id, {"endpoints": connection_strings}) + self.update_relation_data(relation_id, {"endpoints": connection_strings}) def set_read_only_endpoints(self, relation_id: int, connection_strings: str) -> None: """Set database replicas connection strings. @@ -779,70 +2569,174 @@ def set_read_only_endpoints(self, relation_id: int, connection_strings: str) -> relation_id: the identifier for a particular relation. connection_strings: database hosts and ports comma separated list. """ - self._update_relation_data(relation_id, {"read-only-endpoints": connection_strings}) + self.update_relation_data(relation_id, {"read-only-endpoints": connection_strings}) def set_replset(self, relation_id: int, replset: str) -> None: """Set replica set name in the application relation databag. - MongoDB only. + MongoDB only. + + Args: + relation_id: the identifier for a particular relation. + replset: replica set name. + """ + self.update_relation_data(relation_id, {"replset": replset}) + + def set_uris(self, relation_id: int, uris: str) -> None: + """Set the database connection URIs in the application relation databag. + + MongoDB, Redis, and OpenSearch only. + + Args: + relation_id: the identifier for a particular relation. + uris: connection URIs. + """ + self.update_relation_data(relation_id, {"uris": uris}) + + def set_version(self, relation_id: int, version: str) -> None: + """Set the database version in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + version: database version. + """ + self.update_relation_data(relation_id, {"version": version}) + + +class DatabaseProviderEventHandlers(EventHandlers): + """Provider-side of the database relation handlers.""" + + on = DatabaseProvidesEvents() # pyright: ignore [reportAssignmentType] + + def __init__( + self, charm: CharmBase, relation_data: DatabaseProviderData, unique_key: str = "" + ): + """Manager of base client relations.""" + super().__init__(charm, relation_data, unique_key) + # Just to calm down pyright, it can't parse that the same type is being used in the super() call above + self.relation_data = relation_data + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Emit a database requested event if the setup key (database name and optional + # extra user roles) was added to the relation databag by the application. + if "database" in diff.added: + getattr(self.on, "database_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + +class DatabaseProvides(DatabaseProviderData, DatabaseProviderEventHandlers): + """Provider-side of the database relations.""" + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + DatabaseProviderData.__init__(self, charm.model, relation_name) + DatabaseProviderEventHandlers.__init__(self, charm, self) + + +class DatabaseRequirerData(RequirerData): + """Requirer-side of the database relation.""" + + def __init__( + self, + model: Model, + relation_name: str, + database_name: str, + extra_user_roles: Optional[str] = None, + relations_aliases: Optional[List[str]] = None, + additional_secret_fields: Optional[List[str]] = [], + external_node_connectivity: bool = False, + ): + """Manager of database client relations.""" + super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) + self.database = database_name + self.relations_aliases = relations_aliases + self.external_node_connectivity = external_node_connectivity + + def is_postgresql_plugin_enabled(self, plugin: str, relation_index: int = 0) -> bool: + """Returns whether a plugin is enabled in the database. Args: - relation_id: the identifier for a particular relation. - replset: replica set name. + plugin: name of the plugin to check. + relation_index: optional relation index to check the database + (default: 0 - first relation). + + PostgreSQL only. """ - self._update_relation_data(relation_id, {"replset": replset}) + # Psycopg 3 is imported locally to avoid the need of its package installation + # when relating to a database charm other than PostgreSQL. + import psycopg - def set_uris(self, relation_id: int, uris: str) -> None: - """Set the database connection URIs in the application relation databag. + # Return False if no relation is established. + if len(self.relations) == 0: + return False - MongoDB, Redis, and OpenSearch only. + relation_id = self.relations[relation_index].id + host = self.fetch_relation_field(relation_id, "endpoints") - Args: - relation_id: the identifier for a particular relation. - uris: connection URIs. - """ - self._update_relation_data(relation_id, {"uris": uris}) + # Return False if there is no endpoint available. + if host is None: + return False - def set_version(self, relation_id: int, version: str) -> None: - """Set the database version in the application relation databag. + host = host.split(":")[0] - Args: - relation_id: the identifier for a particular relation. - version: database version. - """ - self._update_relation_data(relation_id, {"version": version}) + content = self.fetch_relation_data([relation_id], ["username", "password"]).get( + relation_id, {} + ) + user = content.get("username") + password = content.get("password") + + connection_string = ( + f"host='{host}' dbname='{self.database}' user='{user}' password='{password}'" + ) + try: + with psycopg.connect(connection_string) as connection: + with connection.cursor() as cursor: + cursor.execute( + "SELECT TRUE FROM pg_extension WHERE extname=%s::text;", (plugin,) + ) + return cursor.fetchone() is not None + except psycopg.Error as e: + logger.exception( + f"failed to check whether {plugin} plugin is enabled in the database: %s", str(e) + ) + return False -class DatabaseRequires(DataRequires): - """Requires-side of the database relation.""" +class DatabaseRequirerEventHandlers(RequirerEventHandlers): + """Requires-side of the relation.""" - on = DatabaseRequiresEvents() + on = DatabaseRequiresEvents() # pyright: ignore [reportAssignmentType] def __init__( - self, - charm, - relation_name: str, - database_name: str, - extra_user_roles: str = None, - relations_aliases: List[str] = None, + self, charm: CharmBase, relation_data: DatabaseRequirerData, unique_key: str = "" ): - """Manager of database client relations.""" - super().__init__(charm, relation_name, extra_user_roles) - self.database = database_name - self.relations_aliases = relations_aliases + """Manager of base client relations.""" + super().__init__(charm, relation_data, unique_key) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data # Define custom event names for each alias. - if relations_aliases: + if self.relation_data.relations_aliases: # Ensure the number of aliases does not exceed the maximum # of connections allowed in the specific relation. - relation_connection_limit = self.charm.meta.requires[relation_name].limit - if len(relations_aliases) != relation_connection_limit: + relation_connection_limit = self.charm.meta.requires[ + self.relation_data.relation_name + ].limit + if len(self.relation_data.relations_aliases) != relation_connection_limit: raise ValueError( f"The number of aliases must match the maximum number of connections allowed in the relation. " - f"Expected {relation_connection_limit}, got {len(relations_aliases)}" + f"Expected {relation_connection_limit}, got {len(self.relation_data.relations_aliases)}" ) - for relation_alias in relations_aliases: + if self.relation_data.relations_aliases: + for relation_alias in self.relation_data.relations_aliases: self.on.define_event(f"{relation_alias}_database_created", DatabaseCreatedEvent) self.on.define_event( f"{relation_alias}_endpoints_changed", DatabaseEndpointsChangedEvent @@ -852,6 +2746,10 @@ def __init__( DatabaseReadOnlyEndpointsChangedEvent, ) + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + pass + def _assign_relation_alias(self, relation_id: int) -> None: """Assigns an alias to a relation. @@ -861,29 +2759,32 @@ def _assign_relation_alias(self, relation_id: int) -> None: relation_id: the identifier for a particular relation. """ # If no aliases were provided, return immediately. - if not self.relations_aliases: + if not self.relation_data.relations_aliases: return # Return if an alias was already assigned to this relation # (like when there are more than one unit joining the relation). - if ( - self.charm.model.get_relation(self.relation_name, relation_id) - .data[self.local_unit] - .get("alias") - ): + relation = self.charm.model.get_relation(self.relation_data.relation_name, relation_id) + if relation and relation.data[self.relation_data.local_unit].get("alias"): return # Retrieve the available aliases (the ones that weren't assigned to any relation). - available_aliases = self.relations_aliases[:] - for relation in self.charm.model.relations[self.relation_name]: - alias = relation.data[self.local_unit].get("alias") + available_aliases = self.relation_data.relations_aliases[:] + for relation in self.charm.model.relations[self.relation_data.relation_name]: + alias = relation.data[self.relation_data.local_unit].get("alias") if alias: logger.debug("Alias %s was already assigned to relation %d", alias, relation.id) available_aliases.remove(alias) # Set the alias in the unit relation databag of the specific relation. - relation = self.charm.model.get_relation(self.relation_name, relation_id) - relation.data[self.local_unit].update({"alias": available_aliases[0]}) + relation = self.charm.model.get_relation(self.relation_data.relation_name, relation_id) + if relation: + relation.data[self.relation_data.local_unit].update({"alias": available_aliases[0]}) + + # We need to set relation alias also on the application level so, + # it will be accessible in show-unit juju command, executed for a consumer application unit + if self.relation_data.local_unit.is_leader(): + self.relation_data.update_relation_data(relation_id, {"alias": available_aliases[0]}) def _emit_aliased_event(self, event: RelationChangedEvent, event_name: str) -> None: """Emit an aliased event to a particular relation if it has an alias. @@ -907,40 +2808,54 @@ def _get_relation_alias(self, relation_id: int) -> Optional[str]: Returns: the relation alias or None if the relation was not found. """ - for relation in self.charm.model.relations[self.relation_name]: + for relation in self.charm.model.relations[self.relation_data.relation_name]: if relation.id == relation_id: - return relation.data[self.local_unit].get("alias") + return relation.data[self.relation_data.local_unit].get("alias") return None - def _on_relation_joined_event(self, event: RelationJoinedEvent) -> None: - """Event emitted when the application joins the database relation.""" + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the database relation is created.""" + super()._on_relation_created_event(event) + # If relations aliases were provided, assign one to the relation. self._assign_relation_alias(event.relation.id) # Sets both database and extra user roles in the relation # if the roles are provided. Otherwise, sets only the database. - if self.extra_user_roles: - self._update_relation_data( - event.relation.id, - { - "database": self.database, - "extra-user-roles": self.extra_user_roles, - }, - ) - else: - self._update_relation_data(event.relation.id, {"database": self.database}) + if not self.relation_data.local_unit.is_leader(): + return + + event_data = {"database": self.relation_data.database} + + if self.relation_data.extra_user_roles: + event_data["extra-user-roles"] = self.relation_data.extra_user_roles + + # set external-node-connectivity field + if self.relation_data.external_node_connectivity: + event_data["external-node-connectivity"] = "true" + + self.relation_data.update_relation_data(event.relation.id, event_data) def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the database relation has changed.""" # Check which data has changed to emit customs events. diff = self._diff(event) + # Register all new secrets with their labels + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) + # Check if the database is created # (the database charm shared the credentials). - if "username" in diff.added and "password" in diff.added: + secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) + if ( + "username" in diff.added and "password" in diff.added + ) or secret_field_user in diff.added: # Emit the default event (the one without an alias). logger.info("database created at %s", datetime.now()) - self.on.database_created.emit(event.relation, app=event.app, unit=event.unit) + getattr(self.on, "database_created").emit( + event.relation, app=event.app, unit=event.unit + ) # Emit the aliased event (if any). self._emit_aliased_event(event, "database_created") @@ -954,7 +2869,9 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: if "endpoints" in diff.added or "endpoints" in diff.changed: # Emit the default event (the one without an alias). logger.info("endpoints changed on %s", datetime.now()) - self.on.endpoints_changed.emit(event.relation, app=event.app, unit=event.unit) + getattr(self.on, "endpoints_changed").emit( + event.relation, app=event.app, unit=event.unit + ) # Emit the aliased event (if any). self._emit_aliased_event(event, "endpoints_changed") @@ -968,7 +2885,7 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: if "read-only-endpoints" in diff.added or "read-only-endpoints" in diff.changed: # Emit the default event (the one without an alias). logger.info("read-only-endpoints changed on %s", datetime.now()) - self.on.read_only_endpoints_changed.emit( + getattr(self.on, "read_only_endpoints_changed").emit( event.relation, app=event.app, unit=event.unit ) @@ -976,7 +2893,37 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: self._emit_aliased_event(event, "read_only_endpoints_changed") -# Kafka related events +class DatabaseRequires(DatabaseRequirerData, DatabaseRequirerEventHandlers): + """Provider-side of the database relations.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + database_name: str, + extra_user_roles: Optional[str] = None, + relations_aliases: Optional[List[str]] = None, + additional_secret_fields: Optional[List[str]] = [], + external_node_connectivity: bool = False, + ): + DatabaseRequirerData.__init__( + self, + charm.model, + relation_name, + database_name, + extra_user_roles, + relations_aliases, + additional_secret_fields, + external_node_connectivity, + ) + DatabaseRequirerEventHandlers.__init__(self, charm, self) + + +################################################################################ +# Charm-specific Relations Data and Events +################################################################################ + +# Kafka Events class KafkaProvidesEvent(RelationEvent): @@ -985,11 +2932,17 @@ class KafkaProvidesEvent(RelationEvent): @property def topic(self) -> Optional[str]: """Returns the topic that was requested.""" + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("topic") @property def consumer_group_prefix(self) -> Optional[str]: """Returns the consumer-group-prefix that was requested.""" + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("consumer-group-prefix") @@ -1012,21 +2965,33 @@ class KafkaRequiresEvent(RelationEvent): @property def topic(self) -> Optional[str]: """Returns the topic.""" + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("topic") @property def bootstrap_server(self) -> Optional[str]: - """Returns a a comma-seperated list of broker uris.""" + """Returns a comma-separated list of broker uris.""" + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("endpoints") @property def consumer_group_prefix(self) -> Optional[str]: """Returns the consumer-group-prefix.""" + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("consumer-group-prefix") @property def zookeeper_uris(self) -> Optional[str]: """Returns a comma separated list of Zookeeper uris.""" + if not self.relation.app: + return None + return self.relation.data[self.relation.app].get("zookeeper-uris") @@ -1051,27 +3016,11 @@ class KafkaRequiresEvents(CharmEvents): # Kafka Provides and Requires -class KafkaProvides(DataProvides): +class KafkaProvidesData(ProviderData): """Provider-side of the Kafka relation.""" - on = KafkaProvidesEvents() - - def __init__(self, charm: CharmBase, relation_name: str) -> None: - super().__init__(charm, relation_name) - - def _on_relation_changed(self, event: RelationChangedEvent) -> None: - """Event emitted when the relation has changed.""" - # Only the leader should handle this event. - if not self.local_unit.is_leader(): - return - - # Check which data has changed to emit customs events. - diff = self._diff(event) - - # Emit a topic requested event if the setup key (topic name and optional - # extra user roles) was added to the relation databag by the application. - if "topic" in diff.added: - self.on.topic_requested.emit(event.relation, app=event.app, unit=event.unit) + def __init__(self, model: Model, relation_name: str) -> None: + super().__init__(model, relation_name) def set_topic(self, relation_id: int, topic: str) -> None: """Set topic name in the application relation databag. @@ -1080,7 +3029,7 @@ def set_topic(self, relation_id: int, topic: str) -> None: relation_id: the identifier for a particular relation. topic: the topic name. """ - self._update_relation_data(relation_id, {"topic": topic}) + self.update_relation_data(relation_id, {"topic": topic}) def set_bootstrap_server(self, relation_id: int, bootstrap_server: str) -> None: """Set the bootstrap server in the application relation databag. @@ -1089,7 +3038,7 @@ def set_bootstrap_server(self, relation_id: int, bootstrap_server: str) -> None: relation_id: the identifier for a particular relation. bootstrap_server: the bootstrap server address. """ - self._update_relation_data(relation_id, {"endpoints": bootstrap_server}) + self.update_relation_data(relation_id, {"endpoints": bootstrap_server}) def set_consumer_group_prefix(self, relation_id: int, consumer_group_prefix: str) -> None: """Set the consumer group prefix in the application relation databag. @@ -1098,47 +3047,111 @@ def set_consumer_group_prefix(self, relation_id: int, consumer_group_prefix: str relation_id: the identifier for a particular relation. consumer_group_prefix: the consumer group prefix string. """ - self._update_relation_data(relation_id, {"consumer-group-prefix": consumer_group_prefix}) + self.update_relation_data(relation_id, {"consumer-group-prefix": consumer_group_prefix}) def set_zookeeper_uris(self, relation_id: int, zookeeper_uris: str) -> None: """Set the zookeeper uris in the application relation databag. Args: relation_id: the identifier for a particular relation. - zookeeper_uris: comma-seperated list of ZooKeeper server uris. + zookeeper_uris: comma-separated list of ZooKeeper server uris. """ - self._update_relation_data(relation_id, {"zookeeper-uris": zookeeper_uris}) + self.update_relation_data(relation_id, {"zookeeper-uris": zookeeper_uris}) -class KafkaRequires(DataRequires): - """Requires-side of the Kafka relation.""" +class KafkaProvidesEventHandlers(EventHandlers): + """Provider-side of the Kafka relation.""" + + on = KafkaProvidesEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KafkaProvidesData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Emit a topic requested event if the setup key (topic name and optional + # extra user roles) was added to the relation databag by the application. + if "topic" in diff.added: + getattr(self.on, "topic_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + +class KafkaProvides(KafkaProvidesData, KafkaProvidesEventHandlers): + """Provider-side of the Kafka relation.""" + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + KafkaProvidesData.__init__(self, charm.model, relation_name) + KafkaProvidesEventHandlers.__init__(self, charm, self) + - on = KafkaRequiresEvents() +class KafkaRequiresData(RequirerData): + """Requirer-side of the Kafka relation.""" def __init__( self, - charm, + model: Model, relation_name: str, topic: str, extra_user_roles: Optional[str] = None, consumer_group_prefix: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], ): """Manager of Kafka client relations.""" - # super().__init__(charm, relation_name) - super().__init__(charm, relation_name, extra_user_roles) - self.charm = charm + super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) self.topic = topic self.consumer_group_prefix = consumer_group_prefix or "" - def _on_relation_joined_event(self, event: RelationJoinedEvent) -> None: - """Event emitted when the application joins the Kafka relation.""" + @property + def topic(self): + """Topic to use in Kafka.""" + return self._topic + + @topic.setter + def topic(self, value): + # Avoid wildcards + if value == "*": + raise ValueError(f"Error on topic '{value}', cannot be a wildcard.") + self._topic = value + + +class KafkaRequiresEventHandlers(RequirerEventHandlers): + """Requires-side of the Kafka relation.""" + + on = KafkaRequiresEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KafkaRequiresData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the Kafka relation is created.""" + super()._on_relation_created_event(event) + + if not self.relation_data.local_unit.is_leader(): + return + # Sets topic, extra user roles, and "consumer-group-prefix" in the relation relation_data = { f: getattr(self, f.replace("-", "_"), "") for f in ["consumer-group-prefix", "extra-user-roles", "topic"] } - self._update_relation_data(event.relation.id, relation_data) + self.relation_data.update_relation_data(event.relation.id, relation_data) + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + pass def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the Kafka relation has changed.""" @@ -1147,21 +3160,306 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Check if the topic is created # (the Kafka charm shared the credentials). - if "username" in diff.added and "password" in diff.added: + + # Register all new secrets with their labels + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) + + secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) + if ( + "username" in diff.added and "password" in diff.added + ) or secret_field_user in diff.added: # Emit the default event (the one without an alias). logger.info("topic created at %s", datetime.now()) - self.on.topic_created.emit(event.relation, app=event.app, unit=event.unit) + getattr(self.on, "topic_created").emit(event.relation, app=event.app, unit=event.unit) # To avoid unnecessary application restarts do not trigger # “endpoints_changed“ event if “topic_created“ is triggered. return - # Emit an endpoints (bootstap-server) changed event if the Kafka endpoints + # Emit an endpoints (bootstrap-server) changed event if the Kafka endpoints # added or changed this info in the relation databag. if "endpoints" in diff.added or "endpoints" in diff.changed: # Emit the default event (the one without an alias). logger.info("endpoints changed on %s", datetime.now()) - self.on.bootstrap_server_changed.emit( + getattr(self.on, "bootstrap_server_changed").emit( + event.relation, app=event.app, unit=event.unit + ) # here check if this is the right design + return + + +class KafkaRequires(KafkaRequiresData, KafkaRequiresEventHandlers): + """Provider-side of the Kafka relation.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + topic: str, + extra_user_roles: Optional[str] = None, + consumer_group_prefix: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + ) -> None: + KafkaRequiresData.__init__( + self, + charm.model, + relation_name, + topic, + extra_user_roles, + consumer_group_prefix, + additional_secret_fields, + ) + KafkaRequiresEventHandlers.__init__(self, charm, self) + + +# Opensearch related events + + +class OpenSearchProvidesEvent(RelationEvent): + """Base class for OpenSearch events.""" + + @property + def index(self) -> Optional[str]: + """Returns the index that was requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("index") + + +class IndexRequestedEvent(OpenSearchProvidesEvent, ExtraRoleEvent): + """Event emitted when a new index is requested for use on this relation.""" + + +class OpenSearchProvidesEvents(CharmEvents): + """OpenSearch events. + + This class defines the events that OpenSearch can emit. + """ + + index_requested = EventSource(IndexRequestedEvent) + + +class OpenSearchRequiresEvent(DatabaseRequiresEvent): + """Base class for OpenSearch requirer events.""" + + +class IndexCreatedEvent(AuthenticationEvent, OpenSearchRequiresEvent): + """Event emitted when a new index is created for use on this relation.""" + + +class OpenSearchRequiresEvents(CharmEvents): + """OpenSearch events. + + This class defines the events that the opensearch requirer can emit. + """ + + index_created = EventSource(IndexCreatedEvent) + endpoints_changed = EventSource(DatabaseEndpointsChangedEvent) + authentication_updated = EventSource(AuthenticationEvent) + + +# OpenSearch Provides and Requires Objects + + +class OpenSearchProvidesData(ProviderData): + """Provider-side of the OpenSearch relation.""" + + def __init__(self, model: Model, relation_name: str) -> None: + super().__init__(model, relation_name) + + def set_index(self, relation_id: int, index: str) -> None: + """Set the index in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + index: the index as it is _created_ on the provider charm. This needn't match the + requested index, and can be used to present a different index name if, for example, + the requested index is invalid. + """ + self.update_relation_data(relation_id, {"index": index}) + + def set_endpoints(self, relation_id: int, endpoints: str) -> None: + """Set the endpoints in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + endpoints: the endpoint addresses for opensearch nodes. + """ + self.update_relation_data(relation_id, {"endpoints": endpoints}) + + def set_version(self, relation_id: int, version: str) -> None: + """Set the opensearch version in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + version: database version. + """ + self.update_relation_data(relation_id, {"version": version}) + + +class OpenSearchProvidesEventHandlers(EventHandlers): + """Provider-side of the OpenSearch relation.""" + + on = OpenSearchProvidesEvents() # pyright: ignore[reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: OpenSearchProvidesData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Emit an index requested event if the setup key (index name and optional extra user roles) + # have been added to the relation databag by the application. + if "index" in diff.added: + getattr(self.on, "index_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + +class OpenSearchProvides(OpenSearchProvidesData, OpenSearchProvidesEventHandlers): + """Provider-side of the OpenSearch relation.""" + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + OpenSearchProvidesData.__init__(self, charm.model, relation_name) + OpenSearchProvidesEventHandlers.__init__(self, charm, self) + + +class OpenSearchRequiresData(RequirerData): + """Requires data side of the OpenSearch relation.""" + + def __init__( + self, + model: Model, + relation_name: str, + index: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + ): + """Manager of OpenSearch client relations.""" + super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) + self.index = index + + +class OpenSearchRequiresEventHandlers(RequirerEventHandlers): + """Requires events side of the OpenSearch relation.""" + + on = OpenSearchRequiresEvents() # pyright: ignore[reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: OpenSearchRequiresData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the OpenSearch relation is created.""" + super()._on_relation_created_event(event) + + if not self.relation_data.local_unit.is_leader(): + return + + # Sets both index and extra user roles in the relation if the roles are provided. + # Otherwise, sets only the index. + data = {"index": self.relation_data.index} + if self.relation_data.extra_user_roles: + data["extra-user-roles"] = self.relation_data.extra_user_roles + + self.relation_data.update_relation_data(event.relation.id, data) + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + if not event.secret.label: + return + + relation = self.relation_data._relation_from_secret_label(event.secret.label) + if not relation: + logging.info( + f"Received secret {event.secret.label} but couldn't parse, seems irrelevant" + ) + return + + if relation.app == self.charm.app: + logging.info("Secret changed event ignored for Secret Owner") + + remote_unit = None + for unit in relation.units: + if unit.app != self.charm.app: + remote_unit = unit + + logger.info("authentication updated") + getattr(self.on, "authentication_updated").emit( + relation, app=relation.app, unit=remote_unit + ) + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the OpenSearch relation has changed. + + This event triggers individual custom events depending on the changing relation. + """ + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Register all new secrets with their labels + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) + + secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) + secret_field_tls = self.relation_data._generate_secret_field_name(SECRET_GROUPS.TLS) + updates = {"username", "password", "tls", "tls-ca", secret_field_user, secret_field_tls} + if len(set(diff._asdict().keys()) - updates) < len(diff): + logger.info("authentication updated at: %s", datetime.now()) + getattr(self.on, "authentication_updated").emit( + event.relation, app=event.app, unit=event.unit + ) + + # Check if the index is created + # (the OpenSearch charm shares the credentials). + if ( + "username" in diff.added and "password" in diff.added + ) or secret_field_user in diff.added: + # Emit the default event (the one without an alias). + logger.info("index created at: %s", datetime.now()) + getattr(self.on, "index_created").emit(event.relation, app=event.app, unit=event.unit) + + # To avoid unnecessary application restarts do not trigger + # “endpoints_changed“ event if “index_created“ is triggered. + return + + # Emit a endpoints changed event if the OpenSearch application added or changed this info + # in the relation databag. + if "endpoints" in diff.added or "endpoints" in diff.changed: + # Emit the default event (the one without an alias). + logger.info("endpoints changed on %s", datetime.now()) + getattr(self.on, "endpoints_changed").emit( event.relation, app=event.app, unit=event.unit ) # here check if this is the right design return + + +class OpenSearchRequires(OpenSearchRequiresData, OpenSearchRequiresEventHandlers): + """Requires-side of the OpenSearch relation.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + index: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + ) -> None: + OpenSearchRequiresData.__init__( + self, + charm.model, + relation_name, + index, + extra_user_roles, + additional_secret_fields, + ) + OpenSearchRequiresEventHandlers.__init__(self, charm, self) diff --git a/tests/integration/ha/continuous_writes.py b/tests/integration/ha/continuous_writes.py index 379dcbe9..847c554a 100644 --- a/tests/integration/ha/continuous_writes.py +++ b/tests/integration/ha/continuous_writes.py @@ -23,9 +23,10 @@ wait_random, ) -from integration.helpers import DUMMY_NAME, get_provider_data +from integration.helpers import APP_NAME, DUMMY_NAME, get_provider_data logger = logging.getLogger(__name__) +logging.getLogger("kafka.conn").disabled = True @dataclass @@ -144,7 +145,7 @@ def _create_process(self): self._process = Process( target=ContinuousWrites._run_async, name="continuous_writes", - args=(self._event, self._queue, 0), + args=(self._event, self._queue, 0, self._ops_test), ) def _stop_process(self): @@ -156,9 +157,9 @@ def _stop_process(self): def _client(self): """Build a Kafka client.""" relation_data = get_provider_data( + ops_test=self._ops_test, unit_name=f"{DUMMY_NAME}/0", - model_full_name=self._ops_test.model_full_name, - endpoint="kafka-client-admin", + owner=APP_NAME, ) return KafkaClient( servers=relation_data["endpoints"].split(","), @@ -168,16 +169,17 @@ def _client(self): ) @staticmethod - async def _run(event: Event, data_queue: Queue, starting_number: int) -> None: # noqa: C901 + async def _run( + event: Event, data_queue: Queue, starting_number: int, ops_test + ) -> None: # noqa: C901 """Continuous writing.""" - initial_data = data_queue.get(True) def _client(): """Build a Kafka client.""" relation_data = get_provider_data( + ops_test=ops_test, unit_name=f"{DUMMY_NAME}/0", - model_full_name=initial_data.model_full_name, - endpoint="kafka-client-admin", + owner=APP_NAME, ) return KafkaClient( servers=relation_data["endpoints"].split(","), @@ -227,6 +229,6 @@ def _produce_message(client: KafkaClient, write_value: int) -> None: time.sleep(0.1) @staticmethod - def _run_async(event: Event, data_queue: Queue, starting_number: int): + def _run_async(event: Event, data_queue: Queue, starting_number: int, ops_test: OpsTest): """Run async code.""" - asyncio.run(ContinuousWrites._run(event, data_queue, starting_number)) + asyncio.run(ContinuousWrites._run(event, data_queue, starting_number, ops_test)) diff --git a/tests/integration/ha/ha_helpers.py b/tests/integration/ha/ha_helpers.py index 498193a9..7c37b42a 100644 --- a/tests/integration/ha/ha_helpers.py +++ b/tests/integration/ha/ha_helpers.py @@ -20,6 +20,7 @@ PROCESS = "kafka.Kafka" SERVICE_DEFAULT_PATH = "/etc/systemd/system/snap.charmed-kafka.daemon.service" +ZK = "zookeeper" logger = logging.getLogger(__name__) @@ -217,9 +218,8 @@ def network_restore(machine_name: str) -> None: def is_up(ops_test: OpsTest, broker_id: int) -> bool: """Return if node up.""" - unit_name = ops_test.model.applications[APP_NAME].units[0].name kafka_zk_relation_data = get_kafka_zk_relation_data( - unit_name=unit_name, model_full_name=ops_test.model_full_name + ops_test=ops_test, owner=ZK, unit_name=f"{APP_NAME}/0" ) active_brokers = get_active_brokers(config=kafka_zk_relation_data) chroot = kafka_zk_relation_data.get("chroot", "") diff --git a/tests/integration/ha/test_ha.py b/tests/integration/ha/test_ha.py index 6ab66747..10bb8c54 100644 --- a/tests/integration/ha/test_ha.py +++ b/tests/integration/ha/test_ha.py @@ -83,10 +83,10 @@ async def test_build_and_deploy(ops_test: OpsTest, kafka_charm, app_charm): assert ops_test.model.applications[ZK_NAME].status == "active" await ops_test.model.add_relation(APP_NAME, ZK_NAME) - async with ops_test.fast_forward(): - await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK_NAME], idle_period=30) - assert ops_test.model.applications[APP_NAME].status == "active" - assert ops_test.model.applications[ZK_NAME].status == "active" + async with ops_test.fast_forward(fast_interval="20s"): + await asyncio.sleep(90) + + await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK_NAME], idle_period=30, status="active") await ops_test.model.add_relation(APP_NAME, f"{DUMMY_NAME}:{REL_NAME_ADMIN}") await ops_test.model.wait_for_idle(apps=[APP_NAME, DUMMY_NAME], idle_period=30) @@ -101,7 +101,7 @@ async def test_replicated_events(ops_test: OpsTest): ) logger.info("Producing messages and checking on all units") produce_and_check_logs( - model_full_name=ops_test.model_full_name, + ops_test=ops_test, kafka_unit_name=f"{APP_NAME}/0", provider_unit_name=f"{DUMMY_NAME}/0", topic="replicated-topic", @@ -145,7 +145,7 @@ async def test_multi_cluster_isolation(ops_test: OpsTest, kafka_charm): ) produce_and_check_logs( - model_full_name=ops_test.model_full_name, + ops_test=ops_test, kafka_unit_name=f"{APP_NAME}/0", provider_unit_name=f"{DUMMY_NAME}/0", topic="hot-topic", @@ -156,16 +156,18 @@ async def test_multi_cluster_isolation(ops_test: OpsTest, kafka_charm): # Check that logs are not found on the second cluster with pytest.raises(AssertionError): check_logs( - model_full_name=ops_test.model_full_name, + ops_test=ops_test, kafka_unit_name=f"{second_kafka_name}/0", topic="hot-topic", ) await asyncio.gather( ops_test.juju( - f"remove-application --force --destroy-storage --no-wait {second_kafka_name}" + f"remove-application --force --destroy-storage --no-wait --no-prompt {second_kafka_name}" + ), + ops_test.juju( + f"remove-application --force --destroy-storage --no-wait --no-prompt {second_zk_name}" ), - ops_test.juju(f"remove-application --force --destroy-storage --no-wait {second_zk_name}"), ) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 7e031bff..bd36c2a1 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. +import json import logging -import re import socket import subprocess from contextlib import closing from pathlib import Path from subprocess import PIPE, check_output -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set import yaml from charms.kafka.v0.client import KafkaClient @@ -30,9 +30,9 @@ logger = logging.getLogger(__name__) -def load_acls(model_full_name: str, zookeeper_uri: str) -> Set[Acl]: +def load_acls(model_full_name: str | None, zk_uris: str) -> Set[Acl]: result = check_output( - f"JUJU_MODEL={model_full_name} juju ssh kafka/0 sudo -i 'charmed-kafka.acls --authorizer-properties zookeeper.connect={zookeeper_uri} --list'", + f"JUJU_MODEL={model_full_name} juju ssh kafka/0 sudo -i 'charmed-kafka.acls --authorizer-properties zookeeper.connect={zk_uris} --list'", stderr=PIPE, shell=True, universal_newlines=True, @@ -41,7 +41,7 @@ def load_acls(model_full_name: str, zookeeper_uri: str) -> Set[Acl]: return AuthManager._parse_acls(acls=result) -def load_super_users(model_full_name: str) -> List[str]: +def load_super_users(model_full_name: str | None) -> List[str]: result = check_output( f"JUJU_MODEL={model_full_name} juju ssh kafka/0 sudo -i 'cat /var/snap/charmed-kafka/current/etc/kafka/server.properties'", stderr=PIPE, @@ -57,131 +57,30 @@ def load_super_users(model_full_name: str) -> List[str]: return [] -def check_user(model_full_name: str, username: str, zookeeper_uri: str) -> None: +def check_user(model_full_name: str | None, username: str) -> None: result = check_output( - f"JUJU_MODEL={model_full_name} juju ssh kafka/0 sudo -i 'charmed-kafka.configs --zookeeper {zookeeper_uri} --describe --entity-type users --entity-name {username}'", + f"JUJU_MODEL={model_full_name} juju ssh kafka/0 sudo -i 'charmed-kafka.configs --bootstrap-server localhost:9092 --describe --entity-type users --entity-name {username}' --command-config /var/snap/charmed-kafka/current/etc/kafka/client.properties", stderr=PIPE, shell=True, universal_newlines=True, ) - assert "SCRAM-SHA-512" in result + assert f"SCRAM credential configs for user-principal '{username}' are SCRAM-SHA-512" in result -def get_user(model_full_name: str, username: str, zookeeper_uri: str) -> str: - result = check_output( - f"JUJU_MODEL={model_full_name} juju ssh kafka/0 sudo -i 'charmed-kafka.configs --zookeeper {zookeeper_uri} --describe --entity-type users --entity-name {username}'", - stderr=PIPE, - shell=True, - universal_newlines=True, - ) - return result - - -def show_unit(unit_name: str, model_full_name: str) -> Any: +def get_user(model_full_name: str | None, username: str = "sync") -> str: result = check_output( - f"JUJU_MODEL={model_full_name} juju show-unit {unit_name}", + f"JUJU_MODEL={model_full_name} juju ssh kafka/0 sudo -i 'cat /var/snap/charmed-kafka/current/etc/kafka/server.properties'", stderr=PIPE, shell=True, universal_newlines=True, - ) - - return yaml.safe_load(result) - + ).splitlines() -def get_zookeeper_connection(unit_name: str, model_full_name: str) -> Tuple[List[str], str]: - result = show_unit(unit_name=unit_name, model_full_name=model_full_name) - - relations_info = result[unit_name]["relation-info"] - - usernames = [] - zookeeper_uri = "" - for info in relations_info: - if info["endpoint"] == "cluster": - for key in info["application-data"].keys(): - if re.match(r"(relation\-[\d]+)", key): - usernames.append(key) - if info["endpoint"] == "zookeeper": - zookeeper_uri = info["application-data"]["uris"] - - if zookeeper_uri and usernames: - return usernames, zookeeper_uri - else: - raise Exception("config not found") - - -def get_kafka_zk_relation_data(unit_name: str, model_full_name: str) -> Dict[str, str]: - result = show_unit(unit_name=unit_name, model_full_name=model_full_name) - relations_info = result[unit_name]["relation-info"] - - zk_relation_data = {} - for info in relations_info: - if info["endpoint"] == "zookeeper": - zk_relation_data["chroot"] = info["application-data"]["chroot"] - zk_relation_data["endpoints"] = info["application-data"]["endpoints"] - zk_relation_data["password"] = info["application-data"]["password"] - zk_relation_data["uris"] = info["application-data"]["uris"] - zk_relation_data["username"] = info["application-data"]["username"] - zk_relation_data["tls"] = info["application-data"]["tls"] + for line in result: + if f'required username="{username}"' in line: break - return zk_relation_data - - -def get_provider_data( - unit_name: str, model_full_name: str, endpoint: str = "kafka-client" -) -> Dict[str, str]: - result = show_unit(unit_name=unit_name, model_full_name=model_full_name) - relations_info = result[unit_name]["relation-info"] - logger.info(f"Relation info: {relations_info}") - provider_relation_data = {} - for info in relations_info: - if info["endpoint"] == endpoint: - logger.info(f"Relation data: {info}") - provider_relation_data["username"] = info["application-data"]["username"] - provider_relation_data["password"] = info["application-data"]["password"] - provider_relation_data["endpoints"] = info["application-data"]["endpoints"] - provider_relation_data["zookeeper-uris"] = info["application-data"]["zookeeper-uris"] - provider_relation_data["tls"] = info["application-data"]["tls"] - if "consumer-group-prefix" in info["application-data"]: - provider_relation_data["consumer-group-prefix"] = info["application-data"][ - "consumer-group-prefix" - ] - provider_relation_data["topic"] = info["application-data"]["topic"] - return provider_relation_data - - -def get_active_brokers(config: Dict) -> Set[str]: - """Gets all brokers currently connected to ZooKeeper. - - Args: - config: the relation data provided by ZooKeeper - - Returns: - Set of active broker ids - """ - chroot = config.get("chroot", "") - hosts = config.get("endpoints", "").split(",") - username = config.get("username", "") - password = config.get("password", "") - - zk = ZooKeeperManager(hosts=hosts, username=username, password=password) - path = f"{chroot}/brokers/ids/" - try: - brokers = zk.leader_znodes(path=path) - # auth might not be ready with ZK after relation yet - except (NoNodeError, AuthFailedError, QuorumLeaderNotFoundError) as e: - logger.debug(str(e)) - return set() - - return brokers - - -async def get_address(ops_test: OpsTest, app_name=APP_NAME, unit_num=0) -> str: - """Get the address for a unit.""" - status = await ops_test.model.get_status() # noqa: F821 - address = status["applications"][app_name]["units"][f"{app_name}/{unit_num}"]["public-address"] - return address + return line async def set_password(ops_test: OpsTest, username="sync", password=None, num_unit=0) -> Any: @@ -207,22 +106,24 @@ async def set_tls_private_key(ops_test: OpsTest, key: Optional[str] = None, num_ return (await action.wait()).results -def extract_private_key(data: dict, unit: int = 0) -> Optional[str]: - list_keys = [ - element["local-unit"]["data"]["private-key"] - for element in data[f"{APP_NAME}/{unit}"]["relation-info"] - if element["endpoint"] == "cluster" - ] - return list_keys[0] if len(list_keys) else None +def extract_private_key(ops_test: OpsTest, unit_name: str) -> str | None: + user_secret = get_secret_by_label( + ops_test, + label=f"cluster.{unit_name.split('/')[0]}.unit", + owner=unit_name, + ) + + return user_secret.get("private-key") + +def extract_ca(ops_test: OpsTest, unit_name: str) -> str | None: + user_secret = get_secret_by_label( + ops_test, + label=f"cluster.{unit_name.split('/')[0]}.unit", + owner=unit_name, + ) -def extract_ca(data: dict, unit: int = 0) -> Optional[str]: - list_keys = [ - element["local-unit"]["data"]["ca"] - for element in data[f"{APP_NAME}/{unit}"]["relation-info"] - if element["endpoint"] == "cluster" - ] - return list_keys[0] if len(list_keys) else None + return user_secret.get("ca-cert") or user_secret.get("ca") def check_socket(host: str, port: int) -> bool: @@ -238,27 +139,28 @@ def check_tls(ip: str, port: int) -> bool: shell=True, universal_newlines=True, ) + # FIXME: The server cannot be validated, we would need to try to connect using the CA # from self-signed certificates. This is indication enough that the server is sending a # self-signed key. - return "CN = kafka" in result + return "CN = kafka" in result or "CN=kafka" in result except subprocess.CalledProcessError as e: logger.error(f"command '{e.cmd}' return with error (code {e.returncode}): {e.output}") return False -def consume_and_check(model_full_name: str, provider_unit_name: str, topic: str) -> None: +def consume_and_check(ops_test: OpsTest, provider_unit_name: str, topic: str) -> None: """Consumes 15 messages created by `produce_and_check_logs` function. Args: - model_full_name: the full name of the model + ops_test: OpsTest provider_unit_name: the app to grab credentials from topic: the desired topic to consume from """ relation_data = get_provider_data( + ops_test=ops_test, unit_name=provider_unit_name, - model_full_name=model_full_name, - endpoint="kafka-client-admin", + owner=APP_NAME, ) topic = topic username = relation_data.get("username", None) @@ -283,7 +185,7 @@ def consume_and_check(model_full_name: str, provider_unit_name: str, topic: str) def produce_and_check_logs( - model_full_name: str, + ops_test: OpsTest, kafka_unit_name: str, provider_unit_name: str, topic: str, @@ -294,7 +196,7 @@ def produce_and_check_logs( """Produces 15 messages from HN to chosen Kafka topic. Args: - model_full_name: the full name of the model + ops_test: OpsTest kafka_unit_name: the kafka unit to checks logs on provider_unit_name: the app to grab credentials from topic: the desired topic to produce to @@ -307,9 +209,9 @@ def produce_and_check_logs( AssertionError: if logs aren't found for desired topic """ relation_data = get_provider_data( + ops_test=ops_test, unit_name=provider_unit_name, - model_full_name=model_full_name, - endpoint="kafka-client-admin", + owner=APP_NAME, ) client = KafkaClient( servers=relation_data["endpoints"].split(","), @@ -325,30 +227,29 @@ def produce_and_check_logs( replication_factor=replication_factor, ) client.create_topic(topic=topic_config) + for i in range(TEST_DEFAULT_MESSAGES): message = f"Message #{i}" client.produce_message(topic_name=topic, message_content=message) - check_logs(model_full_name, kafka_unit_name, topic) + check_logs(ops_test, kafka_unit_name, topic) -def check_logs(model_full_name: str, kafka_unit_name: str, topic: str) -> None: +def check_logs(ops_test: OpsTest, kafka_unit_name: str, topic: str) -> None: """Checks if messages for a topic have been produced. Args: - model_full_name: the full name of the model + ops_test: OpsTest kafka_unit_name: the kafka unit to checks logs on topic: the desired topic to check """ logs = check_output( - f"JUJU_MODEL={model_full_name} juju ssh {kafka_unit_name} sudo -i 'find {PATHS['DATA']}/data'", + f"JUJU_MODEL={ops_test.model_full_name} juju ssh {kafka_unit_name} sudo -i 'find {PATHS['DATA']}/data'", stderr=PIPE, shell=True, universal_newlines=True, ).splitlines() - logger.debug(f"{logs=}") - passed = False for log in logs: if topic and "index" in log: @@ -386,7 +287,7 @@ async def set_mtls_client_acls(ops_test: OpsTest, bootstrap_server: str) -> str: return result -def count_lines_with(model_full_name: str, unit: str, file: str, pattern: str) -> int: +def count_lines_with(model_full_name: str | None, unit: str, file: str, pattern: str) -> int: result = check_output( f"JUJU_MODEL={model_full_name} juju ssh {unit} sudo -i 'grep \"{pattern}\" {file} | wc -l'", stderr=PIPE, @@ -395,3 +296,150 @@ def count_lines_with(model_full_name: str, unit: str, file: str, pattern: str) - ) return int(result) + + +def get_secret_by_label(ops_test: OpsTest, label: str, owner: str) -> dict[str, str]: + secrets_meta_raw = check_output( + f"JUJU_MODEL={ops_test.model_full_name} juju list-secrets --format json", + stderr=PIPE, + shell=True, + universal_newlines=True, + ).strip() + secrets_meta = json.loads(secrets_meta_raw) + + for secret_id in secrets_meta: + if owner and not secrets_meta[secret_id]["owner"] == owner: + continue + if secrets_meta[secret_id]["label"] == label: + break + + secrets_data_raw = check_output( + f"JUJU_MODEL={ops_test.model_full_name} juju show-secret --format json --reveal {secret_id}", + stderr=PIPE, + shell=True, + universal_newlines=True, + ) + + secret_data = json.loads(secrets_data_raw) + return secret_data[secret_id]["content"]["Data"] + + +def show_unit(ops_test: OpsTest, unit_name: str) -> Any: + result = check_output( + f"JUJU_MODEL={ops_test.model_full_name} juju show-unit {unit_name}", + stderr=PIPE, + shell=True, + universal_newlines=True, + ) + + return yaml.safe_load(result) + + +def get_client_usernames(ops_test: OpsTest, owner: str = APP_NAME) -> set[str]: + app_secret = get_secret_by_label(ops_test, label=f"cluster.{owner}.app", owner=owner) + + usernames = set() + for key in app_secret.keys(): + if "password" in key: + usernames.add(key.split("-")[0]) + if "relation" in key: + usernames.add(key) + + return usernames + + +# FIXME: will need updating after zookeeper_client is implemented in full +def get_kafka_zk_relation_data( + ops_test: OpsTest, owner: str, unit_name: str, relation_name: str = ZK_NAME +) -> dict[str, str]: + unit_data = show_unit(ops_test, unit_name) + + kafka_zk_relation_data = {} + for info in unit_data[unit_name]["relation-info"]: + if info["endpoint"] == relation_name: + kafka_zk_relation_data["relation-id"] = info["relation-id"] + + # initially collects all non-secret keys + kafka_zk_relation_data.update(dict(info["application-data"])) + + user_secret = get_secret_by_label( + ops_test, + label=f"{relation_name}.{kafka_zk_relation_data['relation-id']}.user.secret", + owner=owner, + ) + + tls_secret = get_secret_by_label( + ops_test, + label=f"{relation_name}.{kafka_zk_relation_data['relation-id']}.tls.secret", + owner=owner, + ) + + # overrides to secret keys if found + return kafka_zk_relation_data | user_secret | tls_secret + + +def get_provider_data( + ops_test: OpsTest, + owner: str, + unit_name: str, + relation_name: str = "kafka-client", + relation_interface: str = "kafka-client-admin", +) -> dict[str, str]: + unit_data = show_unit(ops_test, unit_name) + + provider_relation_data = {} + for info in unit_data[unit_name]["relation-info"]: + if info["endpoint"] == relation_interface: + provider_relation_data["relation-id"] = info["relation-id"] + + # initially collects all non-secret keys + provider_relation_data.update(dict(info["application-data"])) + + user_secret = get_secret_by_label( + ops_test, + label=f"{relation_name}.{provider_relation_data['relation-id']}.user.secret", + owner=owner, + ) + + tls_secret = get_secret_by_label( + ops_test, + label=f"{relation_name}.{provider_relation_data['relation-id']}.tls.secret", + owner=owner, + ) + + # overrides to secret keys if found + return provider_relation_data | user_secret | tls_secret + + +def get_active_brokers(config: Dict) -> Set[str]: + """Gets all brokers currently connected to ZooKeeper. + + Args: + config: the relation data provided by ZooKeeper + + Returns: + Set of active broker ids + """ + chroot = config.get("chroot", "") + hosts = config.get("endpoints", "").split(",") + username = config.get("username", "") + password = config.get("password", "") + + zk = ZooKeeperManager(hosts=hosts, username=username, password=password) + path = f"{chroot}/brokers/ids/" + + try: + brokers = zk.leader_znodes(path=path) + # auth might not be ready with ZK after relation yet + except (NoNodeError, AuthFailedError, QuorumLeaderNotFoundError) as e: + logger.warning(str(e)) + return set() + + return brokers + + +async def get_address(ops_test: OpsTest, app_name=APP_NAME, unit_num=0) -> str: + """Get the address for a unit.""" + status = await ops_test.model.get_status() # noqa: F821 + address = status["applications"][app_name]["units"][f"{app_name}/{unit_num}"]["public-address"] + return address diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 3cd7642d..85fc9a00 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -95,11 +95,10 @@ async def test_build_and_deploy(ops_test: OpsTest, kafka_charm): assert ops_test.model.applications[ZK_NAME].status == "active" await ops_test.model.add_relation(APP_NAME, ZK_NAME) - async with ops_test.fast_forward(): - await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK_NAME]) - - assert ops_test.model.applications[APP_NAME].status == "active" - assert ops_test.model.applications[ZK_NAME].status == "active" + async with ops_test.fast_forward(fast_interval="60s"): + await ops_test.model.wait_for_idle( + apps=[APP_NAME, ZK_NAME], idle_period=30, status="active" + ) @pytest.mark.abort_on_fail @@ -118,7 +117,7 @@ async def test_remove_zk_relation_relate(ops_test: OpsTest): assert ops_test.model.applications[ZK_NAME].status == "active" await ops_test.model.add_relation(APP_NAME, ZK_NAME) - async with ops_test.fast_forward(): + async with ops_test.fast_forward(fast_interval="60s"): await ops_test.model.wait_for_idle( apps=[APP_NAME, ZK_NAME], status="active", idle_period=30, timeout=1000 ) @@ -130,24 +129,31 @@ async def test_listeners(ops_test: OpsTest, app_charm): assert check_socket( address, SECURITY_PROTOCOL_PORTS["SASL_PLAINTEXT"].internal ) # Internal listener + # Client listener should not be enabled if there is no relations assert not check_socket(address, SECURITY_PROTOCOL_PORTS["SASL_PLAINTEXT"].client) + # Add relation with dummy app await asyncio.gather( ops_test.model.deploy(app_charm, application_name=DUMMY_NAME, num_units=1, series="jammy"), ) await ops_test.model.wait_for_idle(apps=[APP_NAME, DUMMY_NAME, ZK_NAME]) + await ops_test.model.add_relation(APP_NAME, f"{DUMMY_NAME}:{REL_NAME_ADMIN}") + assert ops_test.model.applications[APP_NAME].status == "active" assert ops_test.model.applications[DUMMY_NAME].status == "active" await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK_NAME, DUMMY_NAME]) + # check that client listener is active assert check_socket(address, SECURITY_PROTOCOL_PORTS["SASL_PLAINTEXT"].client) + # remove relation and check that client listener is not active await ops_test.model.applications[APP_NAME].remove_relation( f"{APP_NAME}:{REL_NAME}", f"{DUMMY_NAME}:{REL_NAME_ADMIN}" ) await ops_test.model.wait_for_idle(apps=[APP_NAME]) + assert not check_socket(address, SECURITY_PROTOCOL_PORTS["SASL_PLAINTEXT"].client) @@ -179,9 +185,10 @@ async def test_logs_write_to_storage(ops_test: OpsTest): time.sleep(10) assert ops_test.model.applications[APP_NAME].status == "active" assert ops_test.model.applications[DUMMY_NAME].status == "active" - await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK_NAME, DUMMY_NAME]) + await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK_NAME, DUMMY_NAME], idle_period=30) + produce_and_check_logs( - model_full_name=ops_test.model_full_name, + ops_test=ops_test, kafka_unit_name=f"{APP_NAME}/0", provider_unit_name=f"{DUMMY_NAME}/0", topic="hot-topic", @@ -259,7 +266,7 @@ async def test_logs_write_to_new_storage(ops_test: OpsTest): time.sleep(5) # to give time for storage to complete produce_and_check_logs( - model_full_name=ops_test.model_full_name, + ops_test=ops_test, kafka_unit_name=f"{APP_NAME}/0", provider_unit_name=f"{DUMMY_NAME}/0", topic="cold-topic", diff --git a/tests/integration/test_password_rotation.py b/tests/integration/test_password_rotation.py index cc46769a..5cc9e785 100644 --- a/tests/integration/test_password_rotation.py +++ b/tests/integration/test_password_rotation.py @@ -8,55 +8,48 @@ import pytest from pytest_operator.plugin import OpsTest -from .helpers import APP_NAME, ZK_NAME, get_kafka_zk_relation_data, get_user, set_password +from .helpers import APP_NAME, ZK_NAME, get_user, set_password logger = logging.getLogger(__name__) +DUMMY_NAME = "app" +REL_NAME_ADMIN = "kafka-client-admin" + @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed -async def test_build_and_deploy(ops_test: OpsTest, kafka_charm): +async def test_build_and_deploy(ops_test: OpsTest, kafka_charm, app_charm): await asyncio.gather( ops_test.model.deploy( ZK_NAME, channel="edge", application_name=ZK_NAME, num_units=3, series="jammy" ), ops_test.model.deploy(kafka_charm, application_name=APP_NAME, num_units=1, series="jammy"), + ops_test.model.deploy(app_charm, application_name=DUMMY_NAME, num_units=1, series="jammy"), ) await ops_test.model.block_until(lambda: len(ops_test.model.applications[ZK_NAME].units) == 3) await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK_NAME]) assert ops_test.model.applications[APP_NAME].status == "blocked" assert ops_test.model.applications[ZK_NAME].status == "active" + # needed to open localhost ports + await ops_test.model.add_relation(APP_NAME, f"{DUMMY_NAME}:{REL_NAME_ADMIN}") await ops_test.model.add_relation(APP_NAME, ZK_NAME) - async with ops_test.fast_forward(): - await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK_NAME]) - - assert ops_test.model.applications[APP_NAME].status == "active" - assert ops_test.model.applications[ZK_NAME].status == "active" + await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK_NAME], status="active", idle_period=30) async def test_password_rotation(ops_test: OpsTest): """Check that password stored on ZK has changed after a password rotation.""" - relation_data = get_kafka_zk_relation_data( - unit_name=f"{APP_NAME}/0", model_full_name=ops_test.model_full_name - ) - uri = relation_data["uris"].split(",")[-1] - initial_sync_user = get_user( - username="sync", - zookeeper_uri=uri, model_full_name=ops_test.model_full_name, ) result = await set_password(ops_test, username="sync", num_unit=0) assert "sync-password" in result.keys() - await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK_NAME]) + await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK_NAME], status="active", idle_period=30) new_sync_user = get_user( - username="sync", - zookeeper_uri=uri, model_full_name=ops_test.model_full_name, ) diff --git a/tests/integration/test_provider.py b/tests/integration/test_provider.py index 31624e90..0fa3a722 100644 --- a/tests/integration/test_provider.py +++ b/tests/integration/test_provider.py @@ -10,8 +10,9 @@ from .helpers import ( check_user, + get_client_usernames, + get_kafka_zk_relation_data, get_provider_data, - get_zookeeper_connection, load_acls, load_super_users, ) @@ -37,39 +38,34 @@ async def test_deploy_charms_relate_active( """Test deploy and relate operations.""" await asyncio.gather( ops_test.model.deploy( - "zookeeper", channel="edge", application_name="zookeeper", num_units=3, series="jammy" + "zookeeper", channel="edge", application_name=ZK, num_units=3, series="jammy" ), ops_test.model.deploy(kafka_charm, application_name=APP_NAME, num_units=1, series="jammy"), ops_test.model.deploy( app_charm, application_name=DUMMY_NAME_1, num_units=1, series="jammy" ), ) - await ops_test.model.wait_for_idle( - apps=[APP_NAME, DUMMY_NAME_1, ZK], timeout=1800, idle_period=30 - ) + await ops_test.model.add_relation(APP_NAME, ZK) - await ops_test.model.wait_for_idle( - apps=[APP_NAME, ZK], status="active", idle_period=30, timeout=1800 - ) await ops_test.model.add_relation(APP_NAME, f"{DUMMY_NAME_1}:{REL_NAME_CONSUMER}") - await ops_test.model.wait_for_idle( - apps=[APP_NAME, DUMMY_NAME_1], status="active", idle_period=30, timeout=1800 - ) - # implicitly tests setting of kafka app data - returned_usernames, zookeeper_uri = get_zookeeper_connection( - unit_name="kafka/0", model_full_name=ops_test.model_full_name - ) - usernames.update(returned_usernames) + async with ops_test.fast_forward(fast_interval="60s"): + await ops_test.model.wait_for_idle( + apps=[APP_NAME, DUMMY_NAME_1, ZK], idle_period=30, status="active" + ) + + usernames.update(get_client_usernames(ops_test)) for username in usernames: check_user( username=username, - zookeeper_uri=zookeeper_uri, model_full_name=ops_test.model_full_name, ) - for acl in load_acls(model_full_name=ops_test.model_full_name, zookeeper_uri=zookeeper_uri): + zk_data = get_kafka_zk_relation_data(ops_test=ops_test, owner=ZK, unit_name=f"{APP_NAME}/0") + zk_uris = zk_data.get("uris", "").split("/")[0] + + for acl in load_acls(model_full_name=ops_test.model_full_name, zk_uris=zk_uris): assert acl.username in usernames assert acl.operation in ["READ", "DESCRIBE"] assert acl.resource_type in ["GROUP", "TOPIC"] @@ -85,26 +81,24 @@ async def test_deploy_multiple_charms_same_topic_relate_active( ): """Test relation with multiple applications.""" await ops_test.model.deploy(app_charm, application_name=DUMMY_NAME_2, num_units=1) - await ops_test.model.wait_for_idle(apps=[DUMMY_NAME_2]) await ops_test.model.add_relation(APP_NAME, f"{DUMMY_NAME_2}:{REL_NAME_CONSUMER}") - await ops_test.model.wait_for_idle(apps=[APP_NAME, DUMMY_NAME_2]) - assert ops_test.model.applications[APP_NAME].status == "active" - assert ops_test.model.applications[DUMMY_NAME_1].status == "active" - assert ops_test.model.applications[DUMMY_NAME_2].status == "active" - returned_usernames, zookeeper_uri = get_zookeeper_connection( - unit_name="kafka/0", model_full_name=ops_test.model_full_name - ) - usernames.update(returned_usernames) + async with ops_test.fast_forward(fast_interval="60s"): + await ops_test.model.wait_for_idle( + apps=[APP_NAME, DUMMY_NAME_1, ZK], idle_period=30, status="active" + ) + usernames.update(get_client_usernames(ops_test)) for username in usernames: check_user( username=username, - zookeeper_uri=zookeeper_uri, model_full_name=ops_test.model_full_name, ) - for acl in load_acls(model_full_name=ops_test.model_full_name, zookeeper_uri=zookeeper_uri): + zk_data = get_kafka_zk_relation_data(ops_test=ops_test, owner=ZK, unit_name=f"{APP_NAME}/0") + zk_uris = zk_data.get("uris", "").split("/")[0] + + for acl in load_acls(model_full_name=ops_test.model_full_name, zk_uris=zk_uris): assert acl.username in usernames assert acl.operation in ["READ", "DESCRIBE"] assert acl.resource_type in ["GROUP", "TOPIC"] @@ -116,27 +110,26 @@ async def test_deploy_multiple_charms_same_topic_relate_active( async def test_remove_application_removes_user_and_acls(ops_test: OpsTest, usernames: set[str]): """Test the correct removal of user and permission after relation removal.""" await ops_test.model.remove_application(DUMMY_NAME_1, block_until_done=True) - await ops_test.model.wait_for_idle(apps=[APP_NAME]) - assert ops_test.model.applications[APP_NAME].status == "active" - _, zookeeper_uri = get_zookeeper_connection( - unit_name="kafka/0", model_full_name=ops_test.model_full_name - ) + async with ops_test.fast_forward(fast_interval="60s"): + await ops_test.model.wait_for_idle(apps=[APP_NAME, ZK], idle_period=30, status="active") # checks that old users are removed from active cluster ACLs - acls = load_acls(model_full_name=ops_test.model_full_name, zookeeper_uri=zookeeper_uri) + zk_data = get_kafka_zk_relation_data(ops_test=ops_test, owner=ZK, unit_name=f"{APP_NAME}/0") + zk_uris = zk_data.get("uris", "").split("/")[0] + + acls = load_acls(model_full_name=ops_test.model_full_name, zk_uris=zk_uris) acl_usernames = set() for acl in acls: acl_usernames.add(acl.username) assert acl_usernames != usernames - # checks that past usernames no longer exist in ZooKeeper + # checks that past usernames no longer exist with pytest.raises(AssertionError): for username in usernames: check_user( username=username, - zookeeper_uri=zookeeper_uri, model_full_name=ops_test.model_full_name, ) @@ -149,23 +142,24 @@ async def test_deploy_producer_same_topic(ops_test: OpsTest, app_charm, username app_charm, application_name=DUMMY_NAME_1, num_units=1, series="jammy" ) ) - await ops_test.model.wait_for_idle(apps=[APP_NAME, DUMMY_NAME_1, ZK]) await ops_test.model.add_relation(APP_NAME, f"{DUMMY_NAME_1}:{REL_NAME_PRODUCER}") - await ops_test.model.wait_for_idle(apps=[APP_NAME, DUMMY_NAME_1]) - assert ops_test.model.applications[APP_NAME].status == "active" - assert ops_test.model.applications[DUMMY_NAME_1].status == "active" + async with ops_test.fast_forward(fast_interval="60s"): + await ops_test.model.wait_for_idle( + apps=[APP_NAME, DUMMY_NAME_1, ZK], idle_period=30, status="active" + ) - returned_usernames, zookeeper_uri = get_zookeeper_connection( - unit_name="kafka/0", model_full_name=ops_test.model_full_name - ) + zk_data = get_kafka_zk_relation_data(ops_test=ops_test, owner=ZK, unit_name=f"{APP_NAME}/0") + zk_uris = zk_data.get("uris", "").split("/")[0] - acls = load_acls(model_full_name=ops_test.model_full_name, zookeeper_uri=zookeeper_uri) + acls = load_acls(model_full_name=ops_test.model_full_name, zk_uris=zk_uris) acl_usernames = set() for acl in acls: acl_usernames.add(acl.username) - usernames.update(returned_usernames) - for acl in load_acls(model_full_name=ops_test.model_full_name, zookeeper_uri=zookeeper_uri): + + usernames.update(get_client_usernames(ops_test)) + + for acl in acls: assert acl.username in usernames assert acl.operation in ["READ", "DESCRIBE", "CREATE", "WRITE"] assert acl.resource_type in ["GROUP", "TOPIC"] @@ -174,8 +168,7 @@ async def test_deploy_producer_same_topic(ops_test: OpsTest, app_charm, username # remove application await ops_test.model.remove_application(DUMMY_NAME_1, block_until_done=True) - await ops_test.model.wait_for_idle(apps=[APP_NAME]) - assert ops_test.model.applications[APP_NAME].status == "active" + await ops_test.model.wait_for_idle(apps=[APP_NAME], idle_period=30, status="active") @pytest.mark.abort_on_fail @@ -197,6 +190,7 @@ async def test_admin_added_to_super_users(ops_test: OpsTest): assert ops_test.model.applications[APP_NAME].status == "active" assert ops_test.model.applications[DUMMY_NAME_1].status == "active" + # check the correct addition of super-users super_users = load_super_users(model_full_name=ops_test.model_full_name) assert len(super_users) == 3 @@ -247,15 +241,16 @@ async def test_connection_updated_on_tls_enabled(ops_test: OpsTest, app_charm): apps=[APP_NAME, ZK, TLS_NAME, DUMMY_NAME_1], timeout=1800, idle_period=60, status="active" ) - assert ops_test.model.applications[APP_NAME].status == "active" - assert ops_test.model.applications[ZK].status == "active" - assert ops_test.model.applications[TLS_NAME].status == "active" + # ensure at least one update-status run + async with ops_test.fast_forward(fast_interval="30s"): + await asyncio.sleep(60) # Check that related application has updated information provider_data = get_provider_data( + ops_test=ops_test, unit_name=f"{DUMMY_NAME_1}/3", - model_full_name=ops_test.model_full_name, - endpoint="kafka-client-consumer", + relation_interface="kafka-client-consumer", + owner=APP_NAME, ) assert provider_data["tls"] == "enabled" diff --git a/tests/integration/test_scaling.py b/tests/integration/test_scaling.py index 4af643ba..093fcd37 100644 --- a/tests/integration/test_scaling.py +++ b/tests/integration/test_scaling.py @@ -4,7 +4,6 @@ import asyncio import logging -import time import pytest from pytest_operator.plugin import OpsTest @@ -34,14 +33,22 @@ async def test_kafka_simple_scale_up(ops_test: OpsTest, kafka_charm): await ops_test.model.applications[CHARM_KEY].add_units(count=2) await ops_test.model.wait_for_idle( - apps=[CHARM_KEY], status="active", timeout=600, idle_period=30, wait_for_exact_units=3 + apps=[CHARM_KEY], status="active", timeout=600, idle_period=20, wait_for_exact_units=3 ) + # ensuring deferred events get cleaned up + async with ops_test.fast_forward(fast_interval="10s"): + await asyncio.sleep(60) + kafka_zk_relation_data = get_kafka_zk_relation_data( - unit_name="kafka/2", model_full_name=ops_test.model_full_name + ops_test=ops_test, + unit_name="kafka/2", + owner=ZK, ) + active_brokers = get_active_brokers(config=kafka_zk_relation_data) chroot = kafka_zk_relation_data.get("chroot", "") + assert f"{chroot}/brokers/ids/0" in active_brokers assert f"{chroot}/brokers/ids/1" in active_brokers assert f"{chroot}/brokers/ids/2" in active_brokers @@ -54,13 +61,19 @@ async def test_kafka_simple_scale_down(ops_test: OpsTest): apps=[CHARM_KEY], status="active", timeout=1000, idle_period=30, wait_for_exact_units=2 ) - time.sleep(30) + # ensuring ZK data gets updated + async with ops_test.fast_forward(fast_interval="20s"): + await asyncio.sleep(60) kafka_zk_relation_data = get_kafka_zk_relation_data( - unit_name="kafka/2", model_full_name=ops_test.model_full_name + ops_test=ops_test, + unit_name="kafka/2", + owner=ZK, ) + active_brokers = get_active_brokers(config=kafka_zk_relation_data) chroot = kafka_zk_relation_data.get("chroot", "") + assert f"{chroot}/brokers/ids/0" in active_brokers assert f"{chroot}/brokers/ids/1" not in active_brokers assert f"{chroot}/brokers/ids/2" in active_brokers diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 802c22b1..a0cb405f 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -29,7 +29,6 @@ get_kafka_zk_relation_data, set_mtls_client_acls, set_tls_private_key, - show_unit, ) from .test_charm import DUMMY_NAME @@ -66,13 +65,11 @@ async def test_deploy_tls(ops_test: OpsTest, kafka_charm): assert ops_test.model.applications[ZK].status == "active" assert ops_test.model.applications[TLS_NAME].status == "active" - # Relate Zookeeper to TLS - async with ops_test.fast_forward(): - await ops_test.model.add_relation(TLS_NAME, ZK) - await ops_test.model.wait_for_idle(apps=[TLS_NAME, ZK], idle_period=15) + await ops_test.model.add_relation(TLS_NAME, ZK) - assert ops_test.model.applications[TLS_NAME].status == "active" - assert ops_test.model.applications[ZK].status == "active" + # Relate Zookeeper to TLS + async with ops_test.fast_forward(fast_interval="60s"): + await ops_test.model.wait_for_idle(apps=[TLS_NAME, ZK], idle_period=30, status="active") @pytest.mark.abort_on_fail @@ -83,7 +80,7 @@ async def test_kafka_tls(ops_test: OpsTest, app_charm): Afterwards, relate Kafka to TLS operator, which unblocks the application. """ # Relate Zookeeper[TLS] to Kafka[Non-TLS] - async with ops_test.fast_forward(): + async with ops_test.fast_forward(fast_interval="60s"): await ops_test.model.add_relation(ZK, CHARM_KEY) await ops_test.model.wait_for_idle( apps=[ZK], idle_period=15, timeout=1000, status="active" @@ -99,39 +96,36 @@ async def test_kafka_tls(ops_test: OpsTest, app_charm): # Extract the key private_key = extract_private_key( - show_unit(f"{CHARM_KEY}/{num_unit}", model_full_name=ops_test.model_full_name), unit=0 + ops_test=ops_test, + unit_name=f"{CHARM_KEY}/{num_unit}", ) - async with ops_test.fast_forward(): - logger.info("Relate Kafka to TLS") - await ops_test.model.add_relation(f"{CHARM_KEY}:{TLS_RELATION}", TLS_NAME) - await ops_test.model.wait_for_idle( - apps=[CHARM_KEY, ZK, TLS_NAME], idle_period=30, timeout=1200, status="active" - ) + # ensuring at least a few update-status + await ops_test.model.add_relation(f"{CHARM_KEY}:{TLS_RELATION}", TLS_NAME) + async with ops_test.fast_forward(fast_interval="20s"): + await asyncio.sleep(60) - assert ops_test.model.applications[CHARM_KEY].status == "active" - assert ops_test.model.applications[ZK].status == "active" + await ops_test.model.wait_for_idle( + apps=[CHARM_KEY, ZK, TLS_NAME], idle_period=30, timeout=1200, status="active" + ) kafka_address = await get_address(ops_test=ops_test, app_name=CHARM_KEY) - logger.info("Check for Kafka TLS") + assert not check_tls(ip=kafka_address, port=SECURITY_PROTOCOL_PORTS["SASL_SSL"].client) - async with ops_test.fast_forward(): - await asyncio.gather( - ops_test.model.deploy( - app_charm, application_name=DUMMY_NAME, num_units=1, series="jammy" - ), - ) - await ops_test.model.wait_for_idle( - apps=[CHARM_KEY, DUMMY_NAME], timeout=1000, idle_period=60 - ) - await ops_test.model.add_relation(CHARM_KEY, f"{DUMMY_NAME}:{REL_NAME_ADMIN}") - await ops_test.model.wait_for_idle( - apps=[CHARM_KEY, DUMMY_NAME], timeout=3600, idle_period=60, status="active" - ) + await asyncio.gather( + ops_test.model.deploy(app_charm, application_name=DUMMY_NAME, num_units=1, series="jammy"), + ) + await ops_test.model.wait_for_idle(apps=[CHARM_KEY, DUMMY_NAME], timeout=1000, idle_period=30) - assert ops_test.model.applications[CHARM_KEY].status == "active" - assert ops_test.model.applications[DUMMY_NAME].status == "active" + # ensuring at least a few update-status + await ops_test.model.add_relation(CHARM_KEY, f"{DUMMY_NAME}:{REL_NAME_ADMIN}") + async with ops_test.fast_forward(fast_interval="20s"): + await asyncio.sleep(60) + + await ops_test.model.wait_for_idle( + apps=[CHARM_KEY, DUMMY_NAME], idle_period=30, status="active" + ) assert check_tls(ip=kafka_address, port=SECURITY_PROTOCOL_PORTS["SASL_SSL"].client) @@ -140,9 +134,14 @@ async def test_kafka_tls(ops_test: OpsTest, app_charm): await set_tls_private_key(ops_test, key=new_private_key) + # ensuring key event actually runs + async with ops_test.fast_forward(fast_interval="10s"): + await asyncio.sleep(60) + # Extract the key private_key_2 = extract_private_key( - show_unit(f"{CHARM_KEY}/{num_unit}", model_full_name=ops_test.model_full_name), unit=0 + ops_test=ops_test, + unit_name=f"{CHARM_KEY}/{num_unit}", ) assert private_key != private_key_2 @@ -172,16 +171,16 @@ async def test_mtls(ops_test: OpsTest): CERTS_NAME, channel="stable", config=tls_config, series="jammy", application_name=MTLS_NAME ) await ops_test.model.wait_for_idle(apps=[MTLS_NAME], timeout=1000, idle_period=15) - async with ops_test.fast_forward(): - await ops_test.model.add_relation( - f"{CHARM_KEY}:{TRUSTED_CERTIFICATE_RELATION}", f"{MTLS_NAME}:{TLS_RELATION}" - ) - await ops_test.model.wait_for_idle( - apps=[CHARM_KEY, MTLS_NAME], idle_period=60, timeout=2000, status="active" - ) + await ops_test.model.add_relation( + f"{CHARM_KEY}:{TRUSTED_CERTIFICATE_RELATION}", f"{MTLS_NAME}:{TLS_RELATION}" + ) + await ops_test.model.wait_for_idle( + apps=[CHARM_KEY, MTLS_NAME], idle_period=60, timeout=2000, status="active" + ) # getting kafka ca and address - broker_ca = extract_ca(show_unit(f"{CHARM_KEY}/0", model_full_name=ops_test.model_full_name)) + broker_ca = extract_ca(ops_test=ops_test, unit_name=f"{CHARM_KEY}/0") + address = await get_address(ops_test, app_name=CHARM_KEY) ssl_port = SECURITY_PROTOCOL_PORTS["SSL"].client sasl_port = SECURITY_PROTOCOL_PORTS["SASL_SSL"].client @@ -250,7 +249,9 @@ async def test_kafka_tls_scaling(ops_test: OpsTest): ) kafka_zk_relation_data = get_kafka_zk_relation_data( - unit_name=f"{CHARM_KEY}/2", model_full_name=ops_test.model_full_name + unit_name=f"{CHARM_KEY}/2", + ops_test=ops_test, + owner=ZK, ) active_brokers = get_active_brokers(config=kafka_zk_relation_data) chroot = kafka_zk_relation_data.get("chroot", "") diff --git a/tests/integration/test_upgrade.py b/tests/integration/test_upgrade.py index 3fd8c25e..e60cdcb5 100644 --- a/tests/integration/test_upgrade.py +++ b/tests/integration/test_upgrade.py @@ -31,14 +31,12 @@ async def test_in_place_upgrade(ops_test: OpsTest, kafka_charm, app_charm): channel="3/edge", application_name=ZK_NAME, num_units=1, - series="jammy", ), ops_test.model.deploy( APP_NAME, application_name=APP_NAME, num_units=1, channel=CHANNEL, - series="jammy", ), ops_test.model.deploy(app_charm, application_name=DUMMY_NAME, num_units=1, series="jammy"), ) @@ -62,7 +60,7 @@ async def test_in_place_upgrade(ops_test: OpsTest, kafka_charm, app_charm): logger.info("Producing messages before upgrading") produce_and_check_logs( - model_full_name=ops_test.model_full_name, + ops_test=ops_test, kafka_unit_name=f"{APP_NAME}/0", provider_unit_name=f"{DUMMY_NAME}/0", topic="hot-topic", @@ -91,7 +89,7 @@ async def test_in_place_upgrade(ops_test: OpsTest, kafka_charm, app_charm): logger.info("Check that produced messages can be consumed afterwards") consume_and_check( - model_full_name=ops_test.model_full_name, + ops_test=ops_test, provider_unit_name=f"{DUMMY_NAME}/0", topic="hot-topic", ) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 63038d8e..d2605af1 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 # Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. -from unittest.mock import patch +from unittest.mock import PropertyMock, patch import pytest +from ops import JujuVersion from src.literals import INTERNAL_USERS, SUBSTRATE @@ -65,3 +66,9 @@ def patched_health_machine_configured(): yield machine_configured else: yield + + +@pytest.fixture(autouse=True) +def juju_has_secrets(mocker): + """Using Juju3 we should always have secrets available.""" + mocker.patch.object(JujuVersion, "has_secrets", new_callable=PropertyMock).return_value = True diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 318fcce6..0a966ff1 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -257,7 +257,7 @@ def test_start_sets_pebble_layer(harness: Harness, zk_data, passwords_data): "group": "kafka", "environment": { "KAFKA_OPTS": " ".join(extra_opts), - "JAVA_HOME": "/usr/lib/jvm/java-17-openjdk-amd64", + "JAVA_HOME": "/usr/lib/jvm/java-18-openjdk-amd64", "LOG_DIR": harness.charm.workload.paths.logs_path, }, } @@ -461,36 +461,6 @@ def test_storage_add_disableenables_and_starts(harness: Harness, zk_data, passwo assert patched_defer.call_count == 0 -@pytest.mark.skipif(SUBSTRATE == "k8s", reason="multiple storage not supported in K8s") -def test_storage_detaching_disableenables_and_starts(harness: Harness, zk_data, passwords_data): - with harness.hooks_disabled(): - peer_rel_id = harness.add_relation(PEER, CHARM_KEY) - harness.add_relation_unit(peer_rel_id, f"{CHARM_KEY}/0") - harness.set_leader(True) - zk_rel_id = harness.add_relation(ZK, ZK) - harness.update_relation_data(zk_rel_id, ZK, zk_data) - harness.update_relation_data(peer_rel_id, CHARM_KEY, passwords_data) - harness.add_storage(storage_name="data", count=2) - harness.attach_storage(storage_id="data/1") - - with ( - patch("workload.KafkaWorkload.active", return_value=True), - patch("charm.KafkaCharm.healthy", return_value=True), - patch("events.upgrade.KafkaUpgrade.idle", return_value=True), - patch("managers.config.KafkaConfigManager.set_server_properties"), - patch("managers.config.KafkaConfigManager.set_client_properties"), - patch("workload.KafkaWorkload.read", return_value=["gandalf=grey"]), - patch("workload.KafkaWorkload.disable_enable") as patched_disable_enable, - patch("workload.KafkaWorkload.start") as patched_start, - patch("ops.framework.EventBase.defer") as patched_defer, - ): - harness.detach_storage(storage_id="data/1") - - assert patched_disable_enable.call_count == 1 - assert patched_start.call_count == 1 - assert patched_defer.call_count == 0 - - def test_zookeeper_changed_sets_passwords_and_creates_users_with_zk(harness: Harness, zk_data): """Checks inter-broker passwords are created on zookeeper-changed hook using zk auth.""" with harness.hooks_disabled(): @@ -562,7 +532,7 @@ def test_zookeeper_broken_cleans_internal_user_credentials(harness: Harness): with ( patch("workload.KafkaWorkload.stop"), patch("workload.KafkaWorkload.exec"), - patch("core.models.StateBase.update") as patched_update, + patch("core.models.KafkaCluster.update") as patched_update, patch( "core.models.KafkaCluster.internal_user_credentials", new_callable=PropertyMock, @@ -643,9 +613,7 @@ def test_config_changed_updates_client_data(harness: Harness): patch("events.upgrade.KafkaUpgrade.idle", return_value=True), patch("workload.KafkaWorkload.read", return_value=["gandalf=white"]), patch("managers.config.KafkaConfigManager.set_zk_jaas_config"), - patch( - "events.provider.KafkaProvider.update_connection_info" - ) as patched_update_connection_info, + patch("charm.KafkaCharm.update_client_data") as patched_update_client_data, patch( "managers.config.KafkaConfigManager.set_client_properties" ) as patched_set_client_properties, @@ -654,7 +622,7 @@ def test_config_changed_updates_client_data(harness: Harness): harness.charm.on.config_changed.emit() patched_set_client_properties.assert_called_once() - patched_update_connection_info.assert_called_once() + patched_update_client_data.assert_called_once() def test_config_changed_restarts(harness: Harness): diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 35701794..d9a71844 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -296,8 +296,8 @@ def test_bootstrap_server(harness: Harness): ) harness.update_relation_data(peer_relation_id, f"{CHARM_KEY}/1", {"private-address": "shelob"}) - assert len(harness.charm.state.bootstrap_server) == 2 - for server in harness.charm.state.bootstrap_server: + assert len(harness.charm.state.bootstrap_server.split(",")) == 2 + for server in harness.charm.state.bootstrap_server.split(","): assert "9092" in server diff --git a/tests/unit/test_health.py b/tests/unit/test_health.py index 14b5b0b3..60812986 100644 --- a/tests/unit/test_health.py +++ b/tests/unit/test_health.py @@ -11,7 +11,7 @@ from ops.testing import Harness from charm import KafkaCharm -from literals import CHARM_KEY, JVM_MEM_MAX_GB, JVM_MEM_MIN_GB +from literals import CHARM_KEY, JVM_MEM_MAX_GB, JVM_MEM_MIN_GB, SUBSTRATE logger = logging.getLogger(__name__) @@ -19,6 +19,8 @@ ACTIONS = str(yaml.safe_load(Path("./actions.yaml").read_text())) METADATA = str(yaml.safe_load(Path("./metadata.yaml").read_text())) +pytestmark = pytest.mark.skipif(SUBSTRATE == "k8s", reason="health checks not used on K8s") + @pytest.fixture def harness(): diff --git a/tests/unit/test_provider.py b/tests/unit/test_provider.py index 53b5ec30..94a561df 100644 --- a/tests/unit/test_provider.py +++ b/tests/unit/test_provider.py @@ -126,7 +126,7 @@ def test_client_relation_joined_sets_necessary_relation_data(harness: Harness): patch("charm.KafkaCharm.healthy", new_callable=PropertyMock, return_value=True), patch("managers.auth.AuthManager.add_user"), patch("workload.KafkaWorkload.run_bin_command"), - patch("core.cluster.ZooKeeper.connect", new_callable=PropertyMock, return_value="yes"), + patch("core.models.ZooKeeper.uris", new_callable=PropertyMock, return_value="yes"), ): harness.set_leader(True) client_rel_id = harness.add_relation(REL_NAME, "app") @@ -136,11 +136,11 @@ def test_client_relation_joined_sets_necessary_relation_data(harness: Harness): client_relation.id, "app", {"topic": "TOPIC", "extra-user-roles": "consumer"} ) harness.add_relation_unit(client_rel_id, "app/0") - logger.info(f"keys: {client_relation.data[harness.charm.app].keys()}") assert sorted( [ "username", "password", + "tls-ca", "endpoints", "data", "zookeeper-uris", diff --git a/tests/unit/test_upgrade.py b/tests/unit/test_upgrade.py index d4a1075a..8ab8b00a 100644 --- a/tests/unit/test_upgrade.py +++ b/tests/unit/test_upgrade.py @@ -14,7 +14,7 @@ from charm import KafkaCharm from events.upgrade import KafkaDependencyModel -from literals import CHARM_KEY, DEPENDENCIES, PEER, ZK +from literals import CHARM_KEY, CONTAINER, DEPENDENCIES, PEER, SUBSTRATE, ZK logger = logging.getLogger(__name__) @@ -24,11 +24,23 @@ METADATA = str(yaml.safe_load(Path("./metadata.yaml").read_text())) +@pytest.fixture() +def upgrade_func() -> str: + if SUBSTRATE == "k8s": + return "_on_kafka_pebble_ready_upgrade" + + return "_on_upgrade_granted" + + @pytest.fixture def harness(zk_data): harness = Harness(KafkaCharm, meta=METADATA, config=CONFIG, actions=ACTIONS) harness.add_relation("restart", CHARM_KEY) harness.add_relation("upgrade", CHARM_KEY) + + if SUBSTRATE == "k8s": + harness.set_can_connect(CONTAINER, True) + peer_rel_id = harness.add_relation(PEER, CHARM_KEY) zk_rel_id = harness.add_relation(ZK, ZK) harness._update_config( @@ -54,10 +66,14 @@ def test_pre_upgrade_check_raises_not_stable(harness: Harness): def test_pre_upgrade_check_succeeds(harness: Harness): - with patch("charm.KafkaCharm.healthy", return_value=True): + with ( + patch("charm.KafkaCharm.healthy", return_value=True), + patch("events.upgrade.KafkaUpgrade._set_rolling_update_partition"), + ): harness.charm.upgrade.pre_upgrade_check() +@pytest.mark.skipif(SUBSTRATE == "k8s", reason="upgrade stack not used on K8s") def test_build_upgrade_stack(harness: Harness): with harness.hooks_disabled(): harness.add_relation_unit(harness.charm.state.peer_relation.id, f"{CHARM_KEY}/1") @@ -108,7 +124,12 @@ def test_kafka_dependency_model(): assert DependencyModel(**value) -def test_upgrade_granted_sets_failed_if_zookeeper_dependency_check_fails(harness: Harness): +def test_upgrade_granted_sets_failed_if_zookeeper_dependency_check_fails( + harness: Harness, upgrade_func: str +): + with harness.hooks_disabled(): + harness.set_leader(True) + with ( patch.object(KazooClient, "start"), patch( @@ -122,13 +143,24 @@ def test_upgrade_granted_sets_failed_if_zookeeper_dependency_check_fails(harness new_callable=PropertyMock, return_value="1.2.3", ), + patch( + "core.cluster.ClusterState.ready_to_start", + new_callable=PropertyMock, + return_value=True, + ), + patch( + "events.upgrade.KafkaUpgrade.idle", + new_callable=PropertyMock, + return_value=False, + ), ): mock_event = MagicMock() - harness.charm.upgrade._on_upgrade_granted(mock_event) + getattr(harness.charm.upgrade, upgrade_func)(mock_event) assert harness.charm.upgrade.state == "failed" +@pytest.mark.skipif(SUBSTRATE == "k8s", reason="Upgrade granted not used on K8s charms") def test_upgrade_granted_sets_failed_if_failed_snap(harness: Harness): with ( patch( @@ -146,43 +178,82 @@ def test_upgrade_granted_sets_failed_if_failed_snap(harness: Harness): assert harness.charm.upgrade.state == "failed" -def test_upgrade_granted_sets_failed_if_failed_upgrade_check(harness: Harness): +def test_upgrade_sets_failed_if_failed_upgrade_check(harness: Harness, upgrade_func: str): with ( patch( - "events.upgrade.KafkaUpgrade.zookeeper_current_version", + "core.models.ZooKeeper.zookeeper_version", new_callable=PropertyMock, - return_value="3.6", + return_value="3.6.2", ), - patch("workload.KafkaWorkload.stop"), + patch("time.sleep"), + patch("managers.config.KafkaConfigManager.set_environment"), + patch("managers.config.KafkaConfigManager.set_server_properties"), + patch("managers.config.KafkaConfigManager.set_zk_jaas_config"), + patch("managers.config.KafkaConfigManager.set_client_properties"), patch("workload.KafkaWorkload.restart") as patched_restart, - patch("workload.KafkaWorkload.install", return_value=True), + patch("workload.KafkaWorkload.start") as patched_start, + patch("workload.KafkaWorkload.stop"), + patch("workload.KafkaWorkload.install"), patch("charm.KafkaCharm.healthy", new_callable=PropertyMock, return_value=False), + patch( + "core.cluster.ClusterState.ready_to_start", + new_callable=PropertyMock, + return_value=True, + ), + patch( + "events.upgrade.KafkaUpgrade.idle", + new_callable=PropertyMock, + return_value=False, + ), ): mock_event = MagicMock() - harness.charm.upgrade._on_upgrade_granted(mock_event) + getattr(harness.charm.upgrade, upgrade_func)(mock_event) - patched_restart.assert_called_once() + assert patched_restart.call_count or patched_start.call_count assert harness.charm.upgrade.state == "failed" -def test_upgrade_granted_succeeds(harness: Harness): +def test_upgrade_succeeds(harness: Harness, upgrade_func: str): with ( patch( - "events.upgrade.KafkaUpgrade.zookeeper_current_version", + "core.models.ZooKeeper.zookeeper_version", new_callable=PropertyMock, - return_value="3.6", + return_value="3.6.2", ), + patch("time.sleep"), + patch("managers.config.KafkaConfigManager.set_environment"), + patch("managers.config.KafkaConfigManager.set_server_properties"), + patch("managers.config.KafkaConfigManager.set_zk_jaas_config"), + patch("managers.config.KafkaConfigManager.set_client_properties"), + patch("workload.KafkaWorkload.restart") as patched_restart, + patch("workload.KafkaWorkload.start") as patched_start, patch("workload.KafkaWorkload.stop"), - patch("workload.KafkaWorkload.restart"), - patch("workload.KafkaWorkload.install", return_value=True), + patch("workload.KafkaWorkload.install"), + patch("workload.KafkaWorkload.active", new_callable=PropertyMock, return_value=True), patch("charm.KafkaCharm.healthy", new_callable=PropertyMock, return_value=True), + patch( + "core.cluster.ClusterState.ready_to_start", + new_callable=PropertyMock, + return_value=True, + ), + patch( + "events.upgrade.KafkaUpgrade.idle", + new_callable=PropertyMock, + return_value=False, + ), + patch( + "core.models.ZooKeeper.broker_active", + return_value=True, + ), ): mock_event = MagicMock() - harness.charm.upgrade._on_upgrade_granted(mock_event) + getattr(harness.charm.upgrade, upgrade_func)(mock_event) + assert patched_restart.call_count or patched_start.call_count assert harness.charm.upgrade.state == "completed" +@pytest.mark.skipif(SUBSTRATE == "k8s", reason="Upgrade granted not used on K8s charms") def test_upgrade_granted_recurses_upgrade_changed_on_leader(harness: Harness): with harness.hooks_disabled(): harness.set_leader(True) @@ -193,6 +264,7 @@ def test_upgrade_granted_recurses_upgrade_changed_on_leader(harness: Harness): new_callable=PropertyMock, return_value="3.6", ), + patch("time.sleep"), patch("workload.KafkaWorkload.stop"), patch("workload.KafkaWorkload.restart"), patch("workload.KafkaWorkload.install", return_value=True), diff --git a/tests/unit/test_workload.py b/tests/unit/test_workload.py index 774cb88d..9f30153d 100644 --- a/tests/unit/test_workload.py +++ b/tests/unit/test_workload.py @@ -5,10 +5,15 @@ from unittest.mock import mock_open, patch import pytest -from charms.operator_libs_linux.v1.snap import SnapError +from literals import SUBSTRATE from workload import KafkaWorkload +if SUBSTRATE == "vm": + from charms.operator_libs_linux.v1.snap import SnapError + +pytestmark = pytest.mark.skipif(SUBSTRATE == "k8s", reason="workload tests not needed for K8s") + def test_run_bin_command_args(patched_exec): """Checks KAFKA_OPTS env-var and zk-tls flag present in all snap commands.""" diff --git a/tox.ini b/tox.ini index 2a9d973d..f4362a60 100644 --- a/tox.ini +++ b/tox.ini @@ -43,7 +43,7 @@ commands = poetry export -f requirements.txt -o requirements.txt --without-hashes poetry install --no-root --only fmt - poetry run ruff --fix {[vars]all_path} + poetry run ruff check --fix {[vars]all_path} poetry run black {[vars]all_path} [testenv:lint] @@ -62,7 +62,7 @@ commands = --skip {tox_root}/poetry.lock poetry run codespell {[vars]lib_path} - poetry run ruff {[vars]all_path} + poetry run ruff check {[vars]all_path} poetry run black --check --diff {[vars]all_path} poetry install --no-root