diff --git a/doc/cephadm/services/smb.rst b/doc/cephadm/services/smb.rst index 15cf085c53d76..abd3f4343f0de 100644 --- a/doc/cephadm/services/smb.rst +++ b/doc/cephadm/services/smb.rst @@ -105,21 +105,43 @@ custom_dns Active Directory even if the Ceph host nodes are not tied into the Active Directory DNS domain(s). -include_ceph_users: +include_ceph_users A list of cephx user (aka entity) names that the Samba Containers may use. The cephx keys for each user in the list will automatically be added to the keyring in the container. -cluster_meta_uri: +cluster_meta_uri A string containing a URI that identifies where the cluster structure metadata will be stored. Required if ``clustered`` feature is set. Must be a RADOS pseudo-URI. -cluster_lock_uri: +cluster_lock_uri A string containing a URI that identifies where Samba/CTDB will store a cluster lock. Required if ``clustered`` feature is set. Must be a RADOS pseudo-URI. +cluster_public_addrs + List of objects; optional. Supported only when using Samba's clustering. + Assign "virtual" IP addresses that will be managed by the clustering + subsystem and may automatically move between nodes running Samba + containers. + Fields: + + address + Required string. An IP address with a required prefix length (example: + ``192.168.4.51/24``). This address will be assigned to one of the + host's network devices and managed automatically. + destination + Optional. String or list of strings. A ``destination`` defines where + the system will assign the managed IPs. Each string value must be a + network address (example ``192.168.4.0/24``). One or more destinations + may be supplied. The typical case is to use exactly one destination and + so the value may be supplied as a string, rather than a list with a + single item. Each destination network will be mapped to a device on a + host. Run ``cephadm list-networks`` for an example of these mappings. + If destination is not supplied the network is automatically determined + using the address value supplied and taken as the destination. + .. note:: diff --git a/doc/mgr/smb.rst b/doc/mgr/smb.rst index 687822c0557e5..05e6369ddf107 100644 --- a/doc/mgr/smb.rst +++ b/doc/mgr/smb.rst @@ -376,6 +376,27 @@ clustering enables clustering regardless of the placement count. A value of ``never`` disables clustering regardless of the placement count. If unspecified, ``default`` is assumed. +public_addrs + List of objects; optional. Supported only when using Samba's clustering. + Assign "virtual" IP addresses that will be managed by the clustering + subsystem and may automatically move between nodes running Samba + containers. + Fields: + + address + Required string. An IP address with a required prefix length (example: + ``192.168.4.51/24``). This address will be assigned to one of the + host's network devices and managed automatically. + destination + Optional. String or list of strings. A ``destination`` defines where + the system will assign the managed IPs. Each string value must be a + network address (example ``192.168.4.0/24``). One or more destinations + may be supplied. The typical case is to use exactly one destination and + so the value may be supplied as a string, rather than a list with a + single item. Each destination network will be mapped to a device on a + host. Run ``cephadm list-networks`` for an example of these mappings. + If destination is not supplied the network is automatically determined + using the address value supplied and taken as the destination. custom_smb_global_options Optional mapping. Specify key-value pairs that will be directly added to the global ``smb.conf`` options (or equivalent) of a Samba server. Do diff --git a/qa/suites/orch/cephadm/smb/tasks/deploy_smb_mgr_ctdb_res_ips.yaml b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_mgr_ctdb_res_ips.yaml new file mode 100644 index 0000000000000..0aa55a53a3d60 --- /dev/null +++ b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_mgr_ctdb_res_ips.yaml @@ -0,0 +1,145 @@ +roles: +# Test is for basic smb deployment & functionality. one node cluster is OK +- - host.a + - mon.a + - mgr.x + - osd.0 + - osd.1 + - client.0 +- - host.b + - mon.b + - osd.2 + - osd.3 +- - host.c + - mon.c + - osd.4 + - osd.5 +# Reserve a host for acting as a domain controller and smb client +- - host.d + - cephadm.exclude +overrides: + ceph: + log-only-match: + - CEPHADM_ +tasks: +- cephadm.deploy_samba_ad_dc: + role: host.d +- vip: + count: 2 +- cephadm: + +- cephadm.shell: + host.a: + - ceph fs volume create cephfs +- cephadm.wait_for_service: + service: mds.cephfs + +- cephadm.shell: + host.a: + # add subvolgroup & subvolumes for test + - cmd: ceph fs subvolumegroup create cephfs smb + - cmd: ceph fs subvolume create cephfs sv1 --group-name=smb --mode=0777 + - cmd: ceph fs subvolume create cephfs sv2 --group-name=smb --mode=0777 + # set up smb cluster and shares + - cmd: ceph mgr module enable smb + # TODO: replace sleep with poll of mgr state? + - cmd: sleep 30 + - cmd: ceph smb apply -i - + stdin: | + # --- Begin Embedded YAML + - resource_type: ceph.smb.cluster + cluster_id: adipctdb + auth_mode: active-directory + domain_settings: + realm: DOMAIN1.SINK.TEST + join_sources: + - source_type: resource + ref: join1-admin + custom_dns: + - "{{ctx.samba_ad_dc_ip}}" + public_addrs: + - address: {{VIP0}}/{{VIPPREFIXLEN}} + - address: {{VIP1}}/{{VIPPREFIXLEN}} + placement: + count: 3 + - resource_type: ceph.smb.join.auth + auth_id: join1-admin + auth: + username: Administrator + password: Passw0rd + - resource_type: ceph.smb.share + cluster_id: adipctdb + share_id: share1 + cephfs: + volume: cephfs + subvolumegroup: smb + subvolume: sv1 + path: / + - resource_type: ceph.smb.share + cluster_id: adipctdb + share_id: share2 + cephfs: + volume: cephfs + subvolumegroup: smb + subvolume: sv2 + path: / + # --- End Embedded YAML +# Wait for the smb service to start +- cephadm.wait_for_service: + service: smb.adipctdb +# Since this is a true cluster there should be a clustermeta in rados +- cephadm.shell: + host.a: + - cmd: rados --pool=.smb -N adipctdb get cluster.meta.json /dev/stdout + +# Check if shares exist +- cephadm.exec: + host.d: + - sleep 30 + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{'host.a'|role_to_remote|attr('ip_address')}}/share1 -c ls" + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{'host.a'|role_to_remote|attr('ip_address')}}/share2 -c ls" + +# verify CTDB is healthy, cluster well formed +- cephadm.exec: + host.a: + - "{{ctx.cephadm}} ls --no-detail | {{ctx.cephadm}} shell jq -r 'map(select(.name | startswith(\"smb.adipctdb\")))[-1].name' > /tmp/svcname" + - "{{ctx.cephadm}} enter -n $(cat /tmp/svcname) ctdb status > /tmp/ctdb_status" + - cat /tmp/ctdb_status + - grep 'pnn:0 .*OK' /tmp/ctdb_status + - grep 'pnn:1 .*OK' /tmp/ctdb_status + - grep 'pnn:2 .*OK' /tmp/ctdb_status + - grep 'Number of nodes:3' /tmp/ctdb_status + - rm -rf /tmp/svcname /tmp/ctdb_status + +# Test the two assigned VIPs +- cephadm.exec: + host.d: + - sleep 30 + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP0}}/share1 -c ls" + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP1}}/share1 -c ls" + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP0}}/share2 -c ls" + - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP1}}/share2 -c ls" + +- cephadm.shell: + host.a: + - cmd: ceph smb apply -i - + stdin: | + # --- Begin Embedded YAML + - resource_type: ceph.smb.cluster + cluster_id: adipctdb + intent: removed + - resource_type: ceph.smb.join.auth + auth_id: join1-admin + intent: removed + - resource_type: ceph.smb.share + cluster_id: adipctdb + share_id: share1 + intent: removed + - resource_type: ceph.smb.share + cluster_id: adipctdb + share_id: share2 + intent: removed + # --- End Embedded YAML +# Wait for the smb service to be removed +- cephadm.wait_for_service_not_present: + service: smb.adipctdb diff --git a/src/cephadm/cephadmlib/daemons/smb.py b/src/cephadm/cephadmlib/daemons/smb.py index 406e9c6964b0c..0aecd418b1bc4 100644 --- a/src/cephadm/cephadmlib/daemons/smb.py +++ b/src/cephadm/cephadmlib/daemons/smb.py @@ -5,7 +5,7 @@ import re import socket -from typing import List, Dict, Tuple, Optional, Any +from typing import List, Dict, Tuple, Optional, Any, NamedTuple from .. import context_getters from .. import daemon_form @@ -27,6 +27,7 @@ from ..daemon_identity import DaemonIdentity, DaemonSubIdentity from ..deploy import DeploymentType from ..exceptions import Error +from ..host_facts import list_networks from ..net_utils import EndPoint @@ -52,6 +53,20 @@ def valid(cls, value: str) -> bool: return False +class ClusterPublicIP(NamedTuple): + address: str + destinations: List[str] + + @classmethod + def convert(cls, item: Dict[str, Any]) -> 'ClusterPublicIP': + assert isinstance(item, dict) + address = item['address'] + assert isinstance(address, str) + destinations = item['destinations'] + assert isinstance(destinations, list) + return cls(address, destinations) + + class Config: identity: DaemonIdentity instance_id: str @@ -92,6 +107,7 @@ def __init__( rank_generation: int = -1, cluster_meta_uri: str = '', cluster_lock_uri: str = '', + cluster_public_addrs: Optional[List[ClusterPublicIP]] = None, ) -> None: self.identity = identity self.instance_id = instance_id @@ -110,6 +126,7 @@ def __init__( self.rank_generation = rank_generation self.cluster_meta_uri = cluster_meta_uri self.cluster_lock_uri = cluster_lock_uri + self.cluster_public_addrs = cluster_public_addrs def __str__(self) -> str: return ( @@ -376,6 +393,7 @@ def __init__(self, ctx: CephadmContext, ident: DaemonIdentity): self._cached_layout: Optional[ContainerLayout] = None self._rank_info = context_getters.fetch_rank_info(ctx) self.smb_port = 445 + self._network_mapper = _NetworkMapper(ctx) logger.debug('Created SMB ContainerDaemonForm instance') @staticmethod @@ -415,6 +433,7 @@ def validate(self) -> None: vhostname = configs.get('virtual_hostname', '') cluster_meta_uri = configs.get('cluster_meta_uri', '') cluster_lock_uri = configs.get('cluster_lock_uri', '') + cluster_public_addrs = configs.get('cluster_public_addrs', []) if not instance_id: raise Error('invalid instance (cluster) id') @@ -432,6 +451,12 @@ def validate(self) -> None: # the cluster/instanced id to the system hostname hname = socket.getfqdn() vhostname = f'{instance_id}-{hname}' + _public_addrs = [ + ClusterPublicIP.convert(v) for v in cluster_public_addrs + ] + if _public_addrs: + # cache the cephadm networks->devices mapping for later + self._network_mapper.load() self._instance_cfg = Config( identity=self._identity, @@ -447,6 +472,7 @@ def validate(self) -> None: vhostname=vhostname, cluster_meta_uri=cluster_meta_uri, cluster_lock_uri=cluster_lock_uri, + cluster_public_addrs=_public_addrs, ) if self._rank_info: ( @@ -674,7 +700,54 @@ def _write_ctdb_stub_config(self, path: pathlib.Path) -> None: 'recovery_lock': f'!{reclock_cmd}', 'cluster_meta_uri': self._cfg.cluster_meta_uri, 'nodes_cmd': nodes_cmd, + 'public_addresses': self._network_mapper.for_sambacc( + self._cfg + ), }, } with file_utils.write_new(path) as fh: json.dump(stub_config, fh) + + +class _NetworkMapper: + """Helper class that maps between cephadm-friendly address-networks + groupings to ctdb-friendly address-device groupings. + """ + + def __init__(self, ctx: CephadmContext): + self._ctx = ctx + self._networks: Dict = {} + + def load(self) -> None: + logger.debug('fetching networks') + self._networks = list_networks(self._ctx) + + def _convert(self, addr: ClusterPublicIP) -> ClusterPublicIP: + devs = [] + for net in addr.destinations: + if net not in self._networks: + # ignore mappings that cant exist on this host + logger.warning( + 'destination network %r not found in %r', + net, + self._networks.keys(), + ) + continue + for dev in self._networks[net]: + logger.debug( + 'adding device %s from network %r for public ip %s', + dev, + net, + addr.address, + ) + devs.append(dev) + return ClusterPublicIP(addr.address, devs) + + def for_sambacc(self, cfg: Config) -> List[Dict[str, Any]]: + if not cfg.cluster_public_addrs: + return [] + addrs = (self._convert(a) for a in (cfg.cluster_public_addrs or [])) + return [ + {'address': a.address, 'interfaces': a.destinations} + for a in addrs + ] diff --git a/src/pybind/mgr/cephadm/services/smb.py b/src/pybind/mgr/cephadm/services/smb.py index 7b6f7497bf1b4..da75136cdfb83 100644 --- a/src/pybind/mgr/cephadm/services/smb.py +++ b/src/pybind/mgr/cephadm/services/smb.py @@ -70,6 +70,9 @@ def generate_config( config_blobs['cluster_meta_uri'] = smb_spec.cluster_meta_uri if smb_spec.cluster_lock_uri: config_blobs['cluster_lock_uri'] = smb_spec.cluster_lock_uri + cluster_public_addrs = smb_spec.strict_cluster_ip_specs() + if cluster_public_addrs: + config_blobs['cluster_public_addrs'] = cluster_public_addrs ceph_users = smb_spec.include_ceph_users or [] config_blobs.update( self._ceph_config_and_keyring_for( diff --git a/src/pybind/mgr/smb/handler.py b/src/pybind/mgr/smb/handler.py index 740f7779af55b..b2285eef57538 100644 --- a/src/pybind/mgr/smb/handler.py +++ b/src/pybind/mgr/smb/handler.py @@ -1205,6 +1205,7 @@ def _generate_smb_service_spec( user_sources=user_sources, custom_dns=cluster.custom_dns, include_ceph_users=user_entities, + cluster_public_addrs=cluster.service_spec_public_addrs(), ) diff --git a/src/pybind/mgr/smb/module.py b/src/pybind/mgr/smb/module.py index 91069c07d573f..1e71721202e80 100644 --- a/src/pybind/mgr/smb/module.py +++ b/src/pybind/mgr/smb/module.py @@ -56,8 +56,6 @@ def __init__(self, *args: str, **kwargs: Any) -> None: authorizer = kwargs.pop('authorizer', None) uo = kwargs.pop('update_orchestration', None) super().__init__(*args, **kwargs) - # the update_orchestration property only works post-init - update_orchestration = self.update_orchestration if uo is None else uo if internal_store is not None: self._internal_store = internal_store log.info('Using internal_store passed to class: {internal_store}') @@ -82,7 +80,7 @@ def __init__(self, *args: str, **kwargs: Any) -> None: public_store=self._public_store, path_resolver=path_resolver, authorizer=authorizer, - orch=(self if update_orchestration else None), + orch=self._orch_backend(enable_orch=uo), ) def _backend_store(self, store_conf: str = '') -> ConfigStore: @@ -111,6 +109,18 @@ def _backend_store(self, store_conf: str = '') -> ConfigStore: return sqlite_store.mgr_sqlite3_db(self, opts) raise ValueError(f'invalid internal store: {name}') + def _orch_backend( + self, enable_orch: Optional[bool] = None + ) -> Optional['Module']: + if enable_orch is not None: + log.info('smb orchestration argument supplied: %r', enable_orch) + return self if enable_orch else None + if self.update_orchestration: + log.warning('smb orchestration enabled by module') + return self + log.warning('smb orchestration is disabled') + return None + @property def update_orchestration(self) -> bool: return cast( diff --git a/src/pybind/mgr/smb/resources.py b/src/pybind/mgr/smb/resources.py index 3f74ed04f2768..d91485f9992bb 100644 --- a/src/pybind/mgr/smb/resources.py +++ b/src/pybind/mgr/smb/resources.py @@ -5,7 +5,11 @@ import yaml -from ceph.deployment.service_spec import PlacementSpec +from ceph.deployment.service_spec import ( + PlacementSpec, + SMBClusterPublicIPSpec, + SpecValidationError, +) from object_format import ErrorResponseBase from . import resourcelib, validation @@ -346,6 +350,25 @@ def to_simplified(self) -> Simplified: return self.to_json() +# This class is a near 1:1 mirror of the service spec helper class. +@resourcelib.component() +class ClusterPublicIPAssignment(_RBase): + address: str + destination: Union[List[str], str, None] = None + + def to_spec(self) -> SMBClusterPublicIPSpec: + return SMBClusterPublicIPSpec( + address=self.address, + destination=self.destination, + ) + + def validate(self) -> None: + try: + self.to_spec().validate() + except SpecValidationError as err: + raise ValueError(str(err)) from err + + @resourcelib.resource('ceph.smb.cluster') class Cluster(_RBase): """Represents a cluster (instance) that is / should be present.""" @@ -361,6 +384,7 @@ class Cluster(_RBase): placement: Optional[WrappedPlacementSpec] = None # control if the cluster is really a cluster clustering: Optional[SMBClustering] = None + public_addrs: Optional[List[ClusterPublicIPAssignment]] = None def validate(self) -> None: if not self.cluster_id: @@ -415,6 +439,13 @@ def is_clustered(self) -> bool: # clustering enabled unless we're deploying a single instance "cluster" return count != 1 + def service_spec_public_addrs( + self, + ) -> Optional[List[SMBClusterPublicIPSpec]]: + if self.public_addrs is None: + return None + return [a.to_spec() for a in self.public_addrs] + @resourcelib.resource('ceph.smb.join.auth') class JoinAuth(_RBase): diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 4b39151632f8a..0ece1c25a8386 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -5,7 +5,7 @@ from collections import OrderedDict from contextlib import contextmanager from functools import wraps -from ipaddress import ip_network, ip_address +from ipaddress import ip_network, ip_address, ip_interface from typing import ( Any, Callable, @@ -2842,6 +2842,126 @@ def validate(self) -> None: yaml.add_representer(CephExporterSpec, ServiceSpec.yaml_representer) +class SMBClusterPublicIPSpec: + # The SMBClusterIPSpec must be able to translate between what cephadm + # knows about the system, networks using network addresses, and what + # ctdb wants, an IP combined with a prefixlen and device names. + def __init__( + self, + address: str, + destination: Union[str, List[str], None] = None, + ) -> None: + self.address = address + self.destination = destination + self.validate() + + def validate(self) -> None: + if not self.address: + raise SpecValidationError('address value missing') + if '/' not in self.address: + raise SpecValidationError( + 'a combined address and prefix length is required' + ) + # in the future we may want to enhance this to take IPs only and figure + # out the prefixlen automatically. However, we going to start simple and + # require prefix lengths just like ctdb itself does. + try: + # cache the parsed interface address internally + self._addr_iface = ip_interface(self.address) + except ValueError as err: + raise SpecValidationError( + f'Cannot parse interface address {self.address}' + ) from err + # we strongly prefer /{prefixlen} form, even if the user supplied + # a netmask + self.address = self._addr_iface.with_prefixlen + + self._destinations = [] + if not self.destination: + return + if isinstance(self.destination, str): + _dests = [self.destination] + elif isinstance(self.destination, list) and all( + isinstance(v, str) for v in self.destination + ): + _dests = self.destination + else: + raise ValueError( + 'destination field must be a string or list of strings' + ) + for dest in _dests: + try: + dnet = ip_network(dest) + except ValueError as err: + raise SpecValidationError( + f'Cannot parse network value {self.address}' + ) from err + self._destinations.append(dnet) + + def __eq__(self, other: Any) -> bool: + try: + return ( + other.address == self.address + and other.destination == self.destination + ) + except AttributeError: + return NotImplemented + + def __repr__(self) -> str: + return ( + f'SMBClusterPublicIPSpec({self.address!r}, {self.destination!r})' + ) + + def to_json(self) -> Dict[str, Any]: + """Return a JSON-compatible representation of the SMBClusterPublicIPSpec.""" + out: Dict[str, Any] = {'address': self.address} + if self.destination: + out['destination'] = self.destination + return out + + def to_strict(self) -> Dict[str, Any]: + """Return a strictly formed expanded JSON-compatible representation of + the spec. This is not round-trip-able. + """ + # The strict form always contains destination as a list of strings. + dests = [n.with_prefixlen for n in self._destinations] + if not dests: + dests = [self._addr_iface.network.with_prefixlen] + return { + 'address': self.address, + 'destinations': dests, + } + + @classmethod + def from_json(cls, spec: Dict[str, Any]) -> 'SMBClusterPublicIPSpec': + if 'address' not in spec: + raise SpecValidationError( + 'SMB cluster public IP spec missing required field: address' + ) + return cls(spec['address'], spec.get('destination')) + + @classmethod + def convert_list( + cls, arg: Optional[List[Any]] + ) -> Optional[List['SMBClusterPublicIPSpec']]: + if arg is None: + return None + assert isinstance(arg, list) + out = [] + for value in arg: + if isinstance(value, cls): + out.append(value) + elif hasattr(value, 'to_json'): + out.append(cls.from_json(value.to_json())) + elif isinstance(value, dict): + out.append(cls.from_json(value)) + else: + raise SpecValidationError( + f"Unknown type for SMBClusterPublicIPSpec: {type(value)}" + ) + return out + + class SMBSpec(ServiceSpec): service_type = 'smb' _valid_features = {'domain', 'clustered'} @@ -2896,6 +3016,10 @@ def __init__( # cluster_lock_uri - a pseudo-uri that resolves to a (rados) object # that will be used by CTDB for a cluster leader / recovery lock. cluster_lock_uri: Optional[str] = None, + # cluster_public_addrs - A list of SMB cluster public IP specs. + # If supplied, these will be used to esatablish floating virtual ips + # managed by Samba CTDB cluster subsystem. + cluster_public_addrs: Optional[List[SMBClusterPublicIPSpec]] = None, # --- genearal tweaks --- extra_container_args: Optional[GeneralArgList] = None, extra_entrypoint_args: Optional[GeneralArgList] = None, @@ -2925,6 +3049,9 @@ def __init__( self.include_ceph_users = include_ceph_users or [] self.cluster_meta_uri = cluster_meta_uri self.cluster_lock_uri = cluster_lock_uri + self.cluster_public_addrs = SMBClusterPublicIPSpec.convert_list( + cluster_public_addrs + ) self.validate() def validate(self) -> None: @@ -2958,6 +3085,8 @@ def validate(self) -> None: raise ValueError( 'cluster lock uri unsupported when "clustered" feature not set' ) + for spec in self.cluster_public_addrs or []: + spec.validate() def _derive_cluster_uri(self, uri: str, objname: str) -> str: if not uri.startswith('rados://'): @@ -2967,5 +3096,17 @@ def _derive_cluster_uri(self, uri: str, objname: str) -> str: uri = 'rados://' + '/'.join(parts) return uri + def strict_cluster_ip_specs(self) -> List[Dict[str, Any]]: + return [s.to_strict() for s in (self.cluster_public_addrs or [])] + + def to_json(self) -> "OrderedDict[str, Any]": + obj = super().to_json() + spec = obj.get('spec') + if spec and spec.get('cluster_public_addrs'): + spec['cluster_public_addrs'] = [ + a.to_json() for a in spec['cluster_public_addrs'] + ] + return obj + yaml.add_representer(SMBSpec, ServiceSpec.yaml_representer)