From 8c845bd4ee8e0fddadcb78b2dba457022df39ff2 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 21 Aug 2024 17:03:19 -0400 Subject: [PATCH 1/8] mgr/smb: simplify orch backend enablement We have a developer/debug module option that allows one to disable triggering orchestration. When I tried to use it I thought it was buggy and I had trouble diagnosing it. The mistake was on my side, but the code change makes it much clearer what is being enabled so I want to keep it. Signed-off-by: John Mulligan --- src/pybind/mgr/smb/module.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/pybind/mgr/smb/module.py b/src/pybind/mgr/smb/module.py index 91069c07d573f..1e71721202e80 100644 --- a/src/pybind/mgr/smb/module.py +++ b/src/pybind/mgr/smb/module.py @@ -56,8 +56,6 @@ def __init__(self, *args: str, **kwargs: Any) -> None: authorizer = kwargs.pop('authorizer', None) uo = kwargs.pop('update_orchestration', None) super().__init__(*args, **kwargs) - # the update_orchestration property only works post-init - update_orchestration = self.update_orchestration if uo is None else uo if internal_store is not None: self._internal_store = internal_store log.info('Using internal_store passed to class: {internal_store}') @@ -82,7 +80,7 @@ def __init__(self, *args: str, **kwargs: Any) -> None: public_store=self._public_store, path_resolver=path_resolver, authorizer=authorizer, - orch=(self if update_orchestration else None), + orch=self._orch_backend(enable_orch=uo), ) def _backend_store(self, store_conf: str = '') -> ConfigStore: @@ -111,6 +109,18 @@ def _backend_store(self, store_conf: str = '') -> ConfigStore: return sqlite_store.mgr_sqlite3_db(self, opts) raise ValueError(f'invalid internal store: {name}') + def _orch_backend( + self, enable_orch: Optional[bool] = None + ) -> Optional['Module']: + if enable_orch is not None: + log.info('smb orchestration argument supplied: %r', enable_orch) + return self if enable_orch else None + if self.update_orchestration: + log.warning('smb orchestration enabled by module') + return self + log.warning('smb orchestration is disabled') + return None + @property def update_orchestration(self) -> bool: return cast( From e6327c51745641057e767b845136831b9ade038b Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 21 Aug 2024 17:03:40 -0400 Subject: [PATCH 2/8] cephadm: add support for cluster public ip addresses to smb daemon When a list of public addresses (and optional network destination(s)) are supplied at deploy time, convert the networks to device names and pass that result to the sambcc ctdb configuration. Signed-off-by: John Mulligan --- src/cephadm/cephadmlib/daemons/smb.py | 75 ++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/src/cephadm/cephadmlib/daemons/smb.py b/src/cephadm/cephadmlib/daemons/smb.py index 406e9c6964b0c..0aecd418b1bc4 100644 --- a/src/cephadm/cephadmlib/daemons/smb.py +++ b/src/cephadm/cephadmlib/daemons/smb.py @@ -5,7 +5,7 @@ import re import socket -from typing import List, Dict, Tuple, Optional, Any +from typing import List, Dict, Tuple, Optional, Any, NamedTuple from .. import context_getters from .. import daemon_form @@ -27,6 +27,7 @@ from ..daemon_identity import DaemonIdentity, DaemonSubIdentity from ..deploy import DeploymentType from ..exceptions import Error +from ..host_facts import list_networks from ..net_utils import EndPoint @@ -52,6 +53,20 @@ def valid(cls, value: str) -> bool: return False +class ClusterPublicIP(NamedTuple): + address: str + destinations: List[str] + + @classmethod + def convert(cls, item: Dict[str, Any]) -> 'ClusterPublicIP': + assert isinstance(item, dict) + address = item['address'] + assert isinstance(address, str) + destinations = item['destinations'] + assert isinstance(destinations, list) + return cls(address, destinations) + + class Config: identity: DaemonIdentity instance_id: str @@ -92,6 +107,7 @@ def __init__( rank_generation: int = -1, cluster_meta_uri: str = '', cluster_lock_uri: str = '', + cluster_public_addrs: Optional[List[ClusterPublicIP]] = None, ) -> None: self.identity = identity self.instance_id = instance_id @@ -110,6 +126,7 @@ def __init__( self.rank_generation = rank_generation self.cluster_meta_uri = cluster_meta_uri self.cluster_lock_uri = cluster_lock_uri + self.cluster_public_addrs = cluster_public_addrs def __str__(self) -> str: return ( @@ -376,6 +393,7 @@ def __init__(self, ctx: CephadmContext, ident: DaemonIdentity): self._cached_layout: Optional[ContainerLayout] = None self._rank_info = context_getters.fetch_rank_info(ctx) self.smb_port = 445 + self._network_mapper = _NetworkMapper(ctx) logger.debug('Created SMB ContainerDaemonForm instance') @staticmethod @@ -415,6 +433,7 @@ def validate(self) -> None: vhostname = configs.get('virtual_hostname', '') cluster_meta_uri = configs.get('cluster_meta_uri', '') cluster_lock_uri = configs.get('cluster_lock_uri', '') + cluster_public_addrs = configs.get('cluster_public_addrs', []) if not instance_id: raise Error('invalid instance (cluster) id') @@ -432,6 +451,12 @@ def validate(self) -> None: # the cluster/instanced id to the system hostname hname = socket.getfqdn() vhostname = f'{instance_id}-{hname}' + _public_addrs = [ + ClusterPublicIP.convert(v) for v in cluster_public_addrs + ] + if _public_addrs: + # cache the cephadm networks->devices mapping for later + self._network_mapper.load() self._instance_cfg = Config( identity=self._identity, @@ -447,6 +472,7 @@ def validate(self) -> None: vhostname=vhostname, cluster_meta_uri=cluster_meta_uri, cluster_lock_uri=cluster_lock_uri, + cluster_public_addrs=_public_addrs, ) if self._rank_info: ( @@ -674,7 +700,54 @@ def _write_ctdb_stub_config(self, path: pathlib.Path) -> None: 'recovery_lock': f'!{reclock_cmd}', 'cluster_meta_uri': self._cfg.cluster_meta_uri, 'nodes_cmd': nodes_cmd, + 'public_addresses': self._network_mapper.for_sambacc( + self._cfg + ), }, } with file_utils.write_new(path) as fh: json.dump(stub_config, fh) + + +class _NetworkMapper: + """Helper class that maps between cephadm-friendly address-networks + groupings to ctdb-friendly address-device groupings. + """ + + def __init__(self, ctx: CephadmContext): + self._ctx = ctx + self._networks: Dict = {} + + def load(self) -> None: + logger.debug('fetching networks') + self._networks = list_networks(self._ctx) + + def _convert(self, addr: ClusterPublicIP) -> ClusterPublicIP: + devs = [] + for net in addr.destinations: + if net not in self._networks: + # ignore mappings that cant exist on this host + logger.warning( + 'destination network %r not found in %r', + net, + self._networks.keys(), + ) + continue + for dev in self._networks[net]: + logger.debug( + 'adding device %s from network %r for public ip %s', + dev, + net, + addr.address, + ) + devs.append(dev) + return ClusterPublicIP(addr.address, devs) + + def for_sambacc(self, cfg: Config) -> List[Dict[str, Any]]: + if not cfg.cluster_public_addrs: + return [] + addrs = (self._convert(a) for a in (cfg.cluster_public_addrs or [])) + return [ + {'address': a.address, 'interfaces': a.destinations} + for a in addrs + ] From e2839803b26e1d375314987df27d8bd28638f6e8 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 21 Aug 2024 11:31:52 -0400 Subject: [PATCH 3/8] python-common/deployment: add a cluster public ip spec for smb This spec can be used to define one or more public addresses that will be automatically assigned to hosts by CTDB. The address can be specified in the "interface" form - an address plus prefix length. Optionally, networks to bind to can be specified. The network value will be converted to a network device name later by cephadm. Signed-off-by: John Mulligan --- .../ceph/deployment/service_spec.py | 143 +++++++++++++++++- 1 file changed, 142 insertions(+), 1 deletion(-) diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 4b39151632f8a..0ece1c25a8386 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -5,7 +5,7 @@ from collections import OrderedDict from contextlib import contextmanager from functools import wraps -from ipaddress import ip_network, ip_address +from ipaddress import ip_network, ip_address, ip_interface from typing import ( Any, Callable, @@ -2842,6 +2842,126 @@ def validate(self) -> None: yaml.add_representer(CephExporterSpec, ServiceSpec.yaml_representer) +class SMBClusterPublicIPSpec: + # The SMBClusterIPSpec must be able to translate between what cephadm + # knows about the system, networks using network addresses, and what + # ctdb wants, an IP combined with a prefixlen and device names. + def __init__( + self, + address: str, + destination: Union[str, List[str], None] = None, + ) -> None: + self.address = address + self.destination = destination + self.validate() + + def validate(self) -> None: + if not self.address: + raise SpecValidationError('address value missing') + if '/' not in self.address: + raise SpecValidationError( + 'a combined address and prefix length is required' + ) + # in the future we may want to enhance this to take IPs only and figure + # out the prefixlen automatically. However, we going to start simple and + # require prefix lengths just like ctdb itself does. + try: + # cache the parsed interface address internally + self._addr_iface = ip_interface(self.address) + except ValueError as err: + raise SpecValidationError( + f'Cannot parse interface address {self.address}' + ) from err + # we strongly prefer /{prefixlen} form, even if the user supplied + # a netmask + self.address = self._addr_iface.with_prefixlen + + self._destinations = [] + if not self.destination: + return + if isinstance(self.destination, str): + _dests = [self.destination] + elif isinstance(self.destination, list) and all( + isinstance(v, str) for v in self.destination + ): + _dests = self.destination + else: + raise ValueError( + 'destination field must be a string or list of strings' + ) + for dest in _dests: + try: + dnet = ip_network(dest) + except ValueError as err: + raise SpecValidationError( + f'Cannot parse network value {self.address}' + ) from err + self._destinations.append(dnet) + + def __eq__(self, other: Any) -> bool: + try: + return ( + other.address == self.address + and other.destination == self.destination + ) + except AttributeError: + return NotImplemented + + def __repr__(self) -> str: + return ( + f'SMBClusterPublicIPSpec({self.address!r}, {self.destination!r})' + ) + + def to_json(self) -> Dict[str, Any]: + """Return a JSON-compatible representation of the SMBClusterPublicIPSpec.""" + out: Dict[str, Any] = {'address': self.address} + if self.destination: + out['destination'] = self.destination + return out + + def to_strict(self) -> Dict[str, Any]: + """Return a strictly formed expanded JSON-compatible representation of + the spec. This is not round-trip-able. + """ + # The strict form always contains destination as a list of strings. + dests = [n.with_prefixlen for n in self._destinations] + if not dests: + dests = [self._addr_iface.network.with_prefixlen] + return { + 'address': self.address, + 'destinations': dests, + } + + @classmethod + def from_json(cls, spec: Dict[str, Any]) -> 'SMBClusterPublicIPSpec': + if 'address' not in spec: + raise SpecValidationError( + 'SMB cluster public IP spec missing required field: address' + ) + return cls(spec['address'], spec.get('destination')) + + @classmethod + def convert_list( + cls, arg: Optional[List[Any]] + ) -> Optional[List['SMBClusterPublicIPSpec']]: + if arg is None: + return None + assert isinstance(arg, list) + out = [] + for value in arg: + if isinstance(value, cls): + out.append(value) + elif hasattr(value, 'to_json'): + out.append(cls.from_json(value.to_json())) + elif isinstance(value, dict): + out.append(cls.from_json(value)) + else: + raise SpecValidationError( + f"Unknown type for SMBClusterPublicIPSpec: {type(value)}" + ) + return out + + class SMBSpec(ServiceSpec): service_type = 'smb' _valid_features = {'domain', 'clustered'} @@ -2896,6 +3016,10 @@ def __init__( # cluster_lock_uri - a pseudo-uri that resolves to a (rados) object # that will be used by CTDB for a cluster leader / recovery lock. cluster_lock_uri: Optional[str] = None, + # cluster_public_addrs - A list of SMB cluster public IP specs. + # If supplied, these will be used to esatablish floating virtual ips + # managed by Samba CTDB cluster subsystem. + cluster_public_addrs: Optional[List[SMBClusterPublicIPSpec]] = None, # --- genearal tweaks --- extra_container_args: Optional[GeneralArgList] = None, extra_entrypoint_args: Optional[GeneralArgList] = None, @@ -2925,6 +3049,9 @@ def __init__( self.include_ceph_users = include_ceph_users or [] self.cluster_meta_uri = cluster_meta_uri self.cluster_lock_uri = cluster_lock_uri + self.cluster_public_addrs = SMBClusterPublicIPSpec.convert_list( + cluster_public_addrs + ) self.validate() def validate(self) -> None: @@ -2958,6 +3085,8 @@ def validate(self) -> None: raise ValueError( 'cluster lock uri unsupported when "clustered" feature not set' ) + for spec in self.cluster_public_addrs or []: + spec.validate() def _derive_cluster_uri(self, uri: str, objname: str) -> str: if not uri.startswith('rados://'): @@ -2967,5 +3096,17 @@ def _derive_cluster_uri(self, uri: str, objname: str) -> str: uri = 'rados://' + '/'.join(parts) return uri + def strict_cluster_ip_specs(self) -> List[Dict[str, Any]]: + return [s.to_strict() for s in (self.cluster_public_addrs or [])] + + def to_json(self) -> "OrderedDict[str, Any]": + obj = super().to_json() + spec = obj.get('spec') + if spec and spec.get('cluster_public_addrs'): + spec['cluster_public_addrs'] = [ + a.to_json() for a in spec['cluster_public_addrs'] + ] + return obj + yaml.add_representer(SMBSpec, ServiceSpec.yaml_representer) From 961fa85174217e847c3a2a59c23b35fa1ef2690e Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 21 Aug 2024 17:02:57 -0400 Subject: [PATCH 4/8] mgr/cephadm: pass public addresses for a cluster to cephadm binary Add the strictly-formed public addresses list as one of the config blobs we pass to the binary for smb container deployment. Signed-off-by: John Mulligan --- src/pybind/mgr/cephadm/services/smb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pybind/mgr/cephadm/services/smb.py b/src/pybind/mgr/cephadm/services/smb.py index 7b6f7497bf1b4..da75136cdfb83 100644 --- a/src/pybind/mgr/cephadm/services/smb.py +++ b/src/pybind/mgr/cephadm/services/smb.py @@ -70,6 +70,9 @@ def generate_config( config_blobs['cluster_meta_uri'] = smb_spec.cluster_meta_uri if smb_spec.cluster_lock_uri: config_blobs['cluster_lock_uri'] = smb_spec.cluster_lock_uri + cluster_public_addrs = smb_spec.strict_cluster_ip_specs() + if cluster_public_addrs: + config_blobs['cluster_public_addrs'] = cluster_public_addrs ceph_users = smb_spec.include_ceph_users or [] config_blobs.update( self._ceph_config_and_keyring_for( From 5d5757e57aaeeb77bc693683b5cd688500cd59c5 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Thu, 22 Aug 2024 14:08:06 -0400 Subject: [PATCH 5/8] mgr/smb: extend cluster resource type to define public ip addrs When a cluster defines public IPs it will pass this information along to the smb service spec. Signed-off-by: John Mulligan --- src/pybind/mgr/smb/resources.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/smb/resources.py b/src/pybind/mgr/smb/resources.py index 3f74ed04f2768..d91485f9992bb 100644 --- a/src/pybind/mgr/smb/resources.py +++ b/src/pybind/mgr/smb/resources.py @@ -5,7 +5,11 @@ import yaml -from ceph.deployment.service_spec import PlacementSpec +from ceph.deployment.service_spec import ( + PlacementSpec, + SMBClusterPublicIPSpec, + SpecValidationError, +) from object_format import ErrorResponseBase from . import resourcelib, validation @@ -346,6 +350,25 @@ def to_simplified(self) -> Simplified: return self.to_json() +# This class is a near 1:1 mirror of the service spec helper class. +@resourcelib.component() +class ClusterPublicIPAssignment(_RBase): + address: str + destination: Union[List[str], str, None] = None + + def to_spec(self) -> SMBClusterPublicIPSpec: + return SMBClusterPublicIPSpec( + address=self.address, + destination=self.destination, + ) + + def validate(self) -> None: + try: + self.to_spec().validate() + except SpecValidationError as err: + raise ValueError(str(err)) from err + + @resourcelib.resource('ceph.smb.cluster') class Cluster(_RBase): """Represents a cluster (instance) that is / should be present.""" @@ -361,6 +384,7 @@ class Cluster(_RBase): placement: Optional[WrappedPlacementSpec] = None # control if the cluster is really a cluster clustering: Optional[SMBClustering] = None + public_addrs: Optional[List[ClusterPublicIPAssignment]] = None def validate(self) -> None: if not self.cluster_id: @@ -415,6 +439,13 @@ def is_clustered(self) -> bool: # clustering enabled unless we're deploying a single instance "cluster" return count != 1 + def service_spec_public_addrs( + self, + ) -> Optional[List[SMBClusterPublicIPSpec]]: + if self.public_addrs is None: + return None + return [a.to_spec() for a in self.public_addrs] + @resourcelib.resource('ceph.smb.join.auth') class JoinAuth(_RBase): From e2649091777f0ea94c8a002e1a02872317111fc6 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Thu, 22 Aug 2024 14:08:16 -0400 Subject: [PATCH 6/8] mgr/smb: add cluster public ip information to service spec Signed-off-by: John Mulligan --- src/pybind/mgr/smb/handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pybind/mgr/smb/handler.py b/src/pybind/mgr/smb/handler.py index 740f7779af55b..b2285eef57538 100644 --- a/src/pybind/mgr/smb/handler.py +++ b/src/pybind/mgr/smb/handler.py @@ -1205,6 +1205,7 @@ def _generate_smb_service_spec( user_sources=user_sources, custom_dns=cluster.custom_dns, include_ceph_users=user_entities, + cluster_public_addrs=cluster.service_spec_public_addrs(), ) From 9997a648d984f8751897fd9adfaa453058da5f14 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Fri, 23 Aug 2024 10:01:08 -0400 Subject: [PATCH 7/8] doc: add documentation for (cluster_)public_addrs options Document the spec and resource options (they're basically the same) for specifying public addresses that will be managed automatically by CTDB. Signed-off-by: John Mulligan --- doc/cephadm/services/smb.rst | 28 +++++++++++++++++++++++++--- doc/mgr/smb.rst | 21 +++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/doc/cephadm/services/smb.rst b/doc/cephadm/services/smb.rst index 15cf085c53d76..abd3f4343f0de 100644 --- a/doc/cephadm/services/smb.rst +++ b/doc/cephadm/services/smb.rst @@ -105,21 +105,43 @@ custom_dns Active Directory even if the Ceph host nodes are not tied into the Active Directory DNS domain(s). -include_ceph_users: +include_ceph_users A list of cephx user (aka entity) names that the Samba Containers may use. The cephx keys for each user in the list will automatically be added to the keyring in the container. -cluster_meta_uri: +cluster_meta_uri A string containing a URI that identifies where the cluster structure metadata will be stored. Required if ``clustered`` feature is set. Must be a RADOS pseudo-URI. -cluster_lock_uri: +cluster_lock_uri A string containing a URI that identifies where Samba/CTDB will store a cluster lock. Required if ``clustered`` feature is set. Must be a RADOS pseudo-URI. +cluster_public_addrs + List of objects; optional. Supported only when using Samba's clustering. + Assign "virtual" IP addresses that will be managed by the clustering + subsystem and may automatically move between nodes running Samba + containers. + Fields: + + address + Required string. An IP address with a required prefix length (example: + ``192.168.4.51/24``). This address will be assigned to one of the + host's network devices and managed automatically. + destination + Optional. String or list of strings. A ``destination`` defines where + the system will assign the managed IPs. Each string value must be a + network address (example ``192.168.4.0/24``). One or more destinations + may be supplied. The typical case is to use exactly one destination and + so the value may be supplied as a string, rather than a list with a + single item. Each destination network will be mapped to a device on a + host. Run ``cephadm list-networks`` for an example of these mappings. + If destination is not supplied the network is automatically determined + using the address value supplied and taken as the destination. + .. note:: diff --git a/doc/mgr/smb.rst b/doc/mgr/smb.rst index 687822c0557e5..05e6369ddf107 100644 --- a/doc/mgr/smb.rst +++ b/doc/mgr/smb.rst @@ -376,6 +376,27 @@ clustering enables clustering regardless of the placement count. A value of ``never`` disables clustering regardless of the placement count. If unspecified, ``default`` is assumed. +public_addrs + List of objects; optional. Supported only when using Samba's clustering. + Assign "virtual" IP addresses that will be managed by the clustering + subsystem and may automatically move between nodes running Samba + containers. + Fields: + + address + Required string. An IP address with a required prefix length (example: + ``192.168.4.51/24``). This address will be assigned to one of the + host's network devices and managed automatically. + destination + Optional. String or list of strings. A ``destination`` defines where + the system will assign the managed IPs. Each string value must be a + network address (example ``192.168.4.0/24``). One or more destinations + may be supplied. The typical case is to use exactly one destination and + so the value may be supplied as a string, rather than a list with a + single item. Each destination network will be mapped to a device on a + host. Run ``cephadm list-networks`` for an example of these mappings. + If destination is not supplied the network is automatically determined + using the address value supplied and taken as the destination. custom_smb_global_options Optional mapping. Specify key-value pairs that will be directly added to the global ``smb.conf`` options (or equivalent) of a Samba server. Do From dc09d17eca9840dd94d0e0692e32f43988ba1ea2 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Fri, 23 Aug 2024 10:16:27 -0400 Subject: [PATCH 8/8] qa/suites/orch: add test for smb with ctdb and cluster public ips Signed-off-by: John Mulligan --- .../tasks/deploy_smb_mgr_ctdb_res_ips.yaml | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 qa/suites/orch/cephadm/smb/tasks/deploy_smb_mgr_ctdb_res_ips.yaml diff --git a/qa/suites/orch/cephadm/smb/tasks/deploy_smb_mgr_ctdb_res_ips.yaml b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_mgr_ctdb_res_ips.yaml new file mode 100644 index 0000000000000..0aa55a53a3d60 --- /dev/null +++ b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_mgr_ctdb_res_ips.yaml @@ -0,0 +1,145 @@ +roles: +# Test is for basic smb deployment & functionality. one node cluster is OK +- - host.a + - mon.a + - mgr.x + - osd.0 + - osd.1 + - client.0 +- - host.b + - mon.b + - osd.2 + - osd.3 +- - host.c + - mon.c + - osd.4 + - osd.5 +# Reserve a host for acting as a domain controller and smb client +- - host.d + - cephadm.exclude +overrides: + ceph: + log-only-match: + - CEPHADM_ +tasks: +- cephadm.deploy_samba_ad_dc: + role: host.d +- vip: + count: 2 +- cephadm: + +- cephadm.shell: + host.a: + - ceph fs volume create cephfs +- cephadm.wait_for_service: + service: mds.cephfs + +- cephadm.shell: + host.a: + # add subvolgroup & subvolumes for test + - cmd: ceph fs subvolumegroup create cephfs smb + - cmd: ceph fs subvolume create cephfs sv1 --group-name=smb --mode=0777 + - cmd: ceph fs subvolume create cephfs sv2 --group-name=smb --mode=0777 + # set up smb cluster and shares + - cmd: ceph mgr module enable smb + # TODO: replace sleep with poll of mgr state? + - cmd: sleep 30 + - cmd: ceph smb apply -i - + stdin: | + # --- Begin Embedded YAML + - resource_type: ceph.smb.cluster + cluster_id: adipctdb + auth_mode: active-directory + domain_settings: + realm: DOMAIN1.SINK.TEST + join_sources: + - source_type: resource + ref: join1-admin + custom_dns: + - "{{ctx.samba_ad_dc_ip}}" + public_addrs: + - address: {{VIP0}}/{{VIPPREFIXLEN}} + - address: {{VIP1}}/{{VIPPREFIXLEN}} + placement: + count: 3 + - resource_type: ceph.smb.join.auth + auth_id: join1-admin + auth: + username: Administrator + password: Passw0rd + - resource_type: ceph.smb.share + cluster_id: adipctdb + share_id: share1 + cephfs: + volume: cephfs + subvolumegroup: smb + subvolume: sv1 + path: / + - resource_type: ceph.smb.share + cluster_id: adipctdb + share_id: share2 + cephfs: + volume: cephfs + subvolumegroup: smb + subvolume: sv2 + path: / + # --- End Embedded YAML +# Wait for the smb service to start +- cephadm.wait_for_service: + service: smb.adipctdb +# Since this is a true cluster there should be a clustermeta in rados +- cephadm.shell: + host.a: + - cmd: rados --pool=.smb -N adipctdb get cluster.meta.json /dev/stdout + +# Check if shares exist +- cephadm.exec: + host.d: + - sleep 30 + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{'host.a'|role_to_remote|attr('ip_address')}}/share1 -c ls" + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{'host.a'|role_to_remote|attr('ip_address')}}/share2 -c ls" + +# verify CTDB is healthy, cluster well formed +- cephadm.exec: + host.a: + - "{{ctx.cephadm}} ls --no-detail | {{ctx.cephadm}} shell jq -r 'map(select(.name | startswith(\"smb.adipctdb\")))[-1].name' > /tmp/svcname" + - "{{ctx.cephadm}} enter -n $(cat /tmp/svcname) ctdb status > /tmp/ctdb_status" + - cat /tmp/ctdb_status + - grep 'pnn:0 .*OK' /tmp/ctdb_status + - grep 'pnn:1 .*OK' /tmp/ctdb_status + - grep 'pnn:2 .*OK' /tmp/ctdb_status + - grep 'Number of nodes:3' /tmp/ctdb_status + - rm -rf /tmp/svcname /tmp/ctdb_status + +# Test the two assigned VIPs +- cephadm.exec: + host.d: + - sleep 30 + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP0}}/share1 -c ls" + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP1}}/share1 -c ls" + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP0}}/share2 -c ls" + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP1}}/share2 -c ls" + +- cephadm.shell: + host.a: + - cmd: ceph smb apply -i - + stdin: | + # --- Begin Embedded YAML + - resource_type: ceph.smb.cluster + cluster_id: adipctdb + intent: removed + - resource_type: ceph.smb.join.auth + auth_id: join1-admin + intent: removed + - resource_type: ceph.smb.share + cluster_id: adipctdb + share_id: share1 + intent: removed + - resource_type: ceph.smb.share + cluster_id: adipctdb + share_id: share2 + intent: removed + # --- End Embedded YAML +# Wait for the smb service to be removed +- cephadm.wait_for_service_not_present: + service: smb.adipctdb