Merge pull request ceph#59419 from phlogistonjohn/jjm-smb-ctdb-vips

smb: cluster public ip addresses support

Reviewed-by: Adam King <[email protected]>
Reviewed-by: Anoop C S <[email protected]>
Reviewed-by: Avan Thakkar <[email protected]>
Reviewed-by: Michael Adam <[email protected]>

doc/cephadm/services/smb.rst

@@ -105,21 +105,43 @@ custom_dns
     Active Directory even if the Ceph host nodes are not tied into the Active
     Directory DNS domain(s).
 
-include_ceph_users:
+include_ceph_users
     A list of cephx user (aka entity) names that the Samba Containers may use.
     The cephx keys for each user in the list will automatically be added to
     the keyring in the container.
 
-cluster_meta_uri:
+cluster_meta_uri
     A string containing a URI that identifies where the cluster structure
     metadata will be stored. Required if ``clustered`` feature is set. Must be
     a RADOS pseudo-URI.
 
-cluster_lock_uri:
+cluster_lock_uri
     A string containing a URI that identifies where Samba/CTDB will store a
     cluster lock. Required if ``clustered`` feature is set. Must be a RADOS
     pseudo-URI.
 
+cluster_public_addrs
+    List of objects; optional. Supported only when using Samba's clustering.
+    Assign "virtual" IP addresses that will be managed by the clustering
+    subsystem and may automatically move between nodes running Samba
+    containers.
+    Fields:
+
+    address
+        Required string. An IP address with a required prefix length (example:
+        ``192.168.4.51/24``). This address will be assigned to one of the
+        host's network devices and managed automatically.
+    destination
+        Optional. String or list of strings. A ``destination`` defines where
+        the system will assign the managed IPs. Each string value must be a
+        network address (example ``192.168.4.0/24``). One or more destinations
+        may be supplied. The typical case is to use exactly one destination and
+        so the value may be supplied as a string, rather than a list with a
+        single item. Each destination network will be mapped to a device on a
+        host. Run ``cephadm list-networks`` for an example of these mappings.
+        If destination is not supplied the network is automatically determined
+        using the address value supplied and taken as the destination.
+
 
 .. note::
 

doc/mgr/smb.rst

@@ -376,6 +376,27 @@ clustering
     enables clustering regardless of the placement count. A value of ``never``
     disables clustering regardless of the placement count. If unspecified,
     ``default`` is assumed.
+public_addrs
+    List of objects; optional. Supported only when using Samba's clustering.
+    Assign "virtual" IP addresses that will be managed by the clustering
+    subsystem and may automatically move between nodes running Samba
+    containers.
+    Fields:
+
+    address
+        Required string. An IP address with a required prefix length (example:
+        ``192.168.4.51/24``). This address will be assigned to one of the
+        host's network devices and managed automatically.
+    destination
+        Optional. String or list of strings. A ``destination`` defines where
+        the system will assign the managed IPs. Each string value must be a
+        network address (example ``192.168.4.0/24``). One or more destinations
+        may be supplied. The typical case is to use exactly one destination and
+        so the value may be supplied as a string, rather than a list with a
+        single item. Each destination network will be mapped to a device on a
+        host. Run ``cephadm list-networks`` for an example of these mappings.
+        If destination is not supplied the network is automatically determined
+        using the address value supplied and taken as the destination.
 custom_smb_global_options
     Optional mapping. Specify key-value pairs that will be directly added to
     the global ``smb.conf`` options (or equivalent) of a Samba server.  Do

qa/suites/orch/cephadm/smb/tasks/deploy_smb_mgr_ctdb_res_ips.yaml

@@ -0,0 +1,145 @@
+roles:
+# Test is for basic smb deployment & functionality. one node cluster is OK
+- - host.a
+  - mon.a
+  - mgr.x
+  - osd.0
+  - osd.1
+  - client.0
+- - host.b
+  - mon.b
+  - osd.2
+  - osd.3
+- - host.c
+  - mon.c
+  - osd.4
+  - osd.5
+# Reserve a host for acting as a domain controller and smb client
+- - host.d
+  - cephadm.exclude
+overrides:
+  ceph:
+    log-only-match:
+      - CEPHADM_
+tasks:
+- cephadm.deploy_samba_ad_dc:
+    role: host.d
+- vip:
+    count: 2
+- cephadm:
+
+- cephadm.shell:
+    host.a:
+      - ceph fs volume create cephfs
+- cephadm.wait_for_service:
+    service: mds.cephfs
+
+- cephadm.shell:
+    host.a:
+      # add subvolgroup & subvolumes for test
+      - cmd: ceph fs subvolumegroup create cephfs smb
+      - cmd: ceph fs subvolume create cephfs sv1 --group-name=smb --mode=0777
+      - cmd: ceph fs subvolume create cephfs sv2 --group-name=smb --mode=0777
+      # set up smb cluster and shares
+      - cmd: ceph mgr module enable smb
+      # TODO: replace sleep with poll of mgr state?
+      - cmd: sleep 30
+      - cmd: ceph smb apply -i -
+        stdin: |
+          # --- Begin Embedded YAML
+          - resource_type: ceph.smb.cluster
+            cluster_id: adipctdb
+            auth_mode: active-directory
+            domain_settings:
+              realm: DOMAIN1.SINK.TEST
+              join_sources:
+                - source_type: resource
+                  ref: join1-admin
+            custom_dns:
+              - "{{ctx.samba_ad_dc_ip}}"
+            public_addrs:
+              - address: {{VIP0}}/{{VIPPREFIXLEN}}
+              - address: {{VIP1}}/{{VIPPREFIXLEN}}
+            placement:
+              count: 3
+          - resource_type: ceph.smb.join.auth
+            auth_id: join1-admin
+            auth:
+              username: Administrator
+              password: Passw0rd
+          - resource_type: ceph.smb.share
+            cluster_id: adipctdb
+            share_id: share1
+            cephfs:
+              volume: cephfs
+              subvolumegroup: smb
+              subvolume: sv1
+              path: /
+          - resource_type: ceph.smb.share
+            cluster_id: adipctdb
+            share_id: share2
+            cephfs:
+              volume: cephfs
+              subvolumegroup: smb
+              subvolume: sv2
+              path: /
+          # --- End Embedded YAML
+# Wait for the smb service to start
+- cephadm.wait_for_service:
+    service: smb.adipctdb
+# Since this is a true cluster there should be a clustermeta in rados
+- cephadm.shell:
+    host.a:
+      - cmd: rados --pool=.smb -N adipctdb get cluster.meta.json /dev/stdout
+
+# Check if shares exist
+- cephadm.exec:
+    host.d:
+      - sleep 30
+      - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{'host.a'|role_to_remote|attr('ip_address')}}/share1 -c ls"
+      - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{'host.a'|role_to_remote|attr('ip_address')}}/share2 -c ls"
+
+# verify CTDB is healthy, cluster well formed
+- cephadm.exec:
+    host.a:
+      - "{{ctx.cephadm}} ls --no-detail  | {{ctx.cephadm}} shell jq -r 'map(select(.name | startswith(\"smb.adipctdb\")))[-1].name' > /tmp/svcname"
+      - "{{ctx.cephadm}} enter -n $(cat /tmp/svcname) ctdb status > /tmp/ctdb_status"
+      - cat /tmp/ctdb_status
+      - grep 'pnn:0 .*OK' /tmp/ctdb_status
+      - grep 'pnn:1 .*OK' /tmp/ctdb_status
+      - grep 'pnn:2 .*OK' /tmp/ctdb_status
+      - grep 'Number of nodes:3' /tmp/ctdb_status
+      - rm -rf /tmp/svcname /tmp/ctdb_status
+
+# Test the two assigned VIPs
+- cephadm.exec:
+    host.d:
+      - sleep 30
+      - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP0}}/share1 -c ls"
+      - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP1}}/share1 -c ls"
+      - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP0}}/share2 -c ls"
+      - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{VIP1}}/share2 -c ls"
+
+- cephadm.shell:
+    host.a:
+      - cmd: ceph smb apply -i -
+        stdin: |
+          # --- Begin Embedded YAML
+          - resource_type: ceph.smb.cluster
+            cluster_id: adipctdb
+            intent: removed
+          - resource_type: ceph.smb.join.auth
+            auth_id: join1-admin
+            intent: removed
+          - resource_type: ceph.smb.share
+            cluster_id: adipctdb
+            share_id: share1
+            intent: removed
+          - resource_type: ceph.smb.share
+            cluster_id: adipctdb
+            share_id: share2
+            intent: removed
+          # --- End Embedded YAML
+# Wait for the smb service to be removed
+- cephadm.wait_for_service_not_present:
+    service: smb.adipctdb

src/cephadm/cephadmlib/daemons/smb.py

@@ -5,7 +5,7 @@
 import re
 import socket
 
-from typing import List, Dict, Tuple, Optional, Any
+from typing import List, Dict, Tuple, Optional, Any, NamedTuple
 
 from .. import context_getters
 from .. import daemon_form
@@ -27,6 +27,7 @@
 from ..daemon_identity import DaemonIdentity, DaemonSubIdentity
 from ..deploy import DeploymentType
 from ..exceptions import Error
+from ..host_facts import list_networks
 from ..net_utils import EndPoint
 
 
@@ -52,6 +53,20 @@ def valid(cls, value: str) -> bool:
             return False
 
 
+class ClusterPublicIP(NamedTuple):
+    address: str
+    destinations: List[str]
+
+    @classmethod
+    def convert(cls, item: Dict[str, Any]) -> 'ClusterPublicIP':
+        assert isinstance(item, dict)
+        address = item['address']
+        assert isinstance(address, str)
+        destinations = item['destinations']
+        assert isinstance(destinations, list)
+        return cls(address, destinations)
+
+
 class Config:
     identity: DaemonIdentity
     instance_id: str
@@ -92,6 +107,7 @@ def __init__(
         rank_generation: int = -1,
         cluster_meta_uri: str = '',
         cluster_lock_uri: str = '',
+        cluster_public_addrs: Optional[List[ClusterPublicIP]] = None,
     ) -> None:
         self.identity = identity
         self.instance_id = instance_id
@@ -110,6 +126,7 @@ def __init__(
         self.rank_generation = rank_generation
         self.cluster_meta_uri = cluster_meta_uri
         self.cluster_lock_uri = cluster_lock_uri
+        self.cluster_public_addrs = cluster_public_addrs
 
     def __str__(self) -> str:
         return (
@@ -376,6 +393,7 @@ def __init__(self, ctx: CephadmContext, ident: DaemonIdentity):
         self._cached_layout: Optional[ContainerLayout] = None
         self._rank_info = context_getters.fetch_rank_info(ctx)
         self.smb_port = 445
+        self._network_mapper = _NetworkMapper(ctx)
         logger.debug('Created SMB ContainerDaemonForm instance')
 
     @staticmethod
@@ -415,6 +433,7 @@ def validate(self) -> None:
         vhostname = configs.get('virtual_hostname', '')
         cluster_meta_uri = configs.get('cluster_meta_uri', '')
         cluster_lock_uri = configs.get('cluster_lock_uri', '')
+        cluster_public_addrs = configs.get('cluster_public_addrs', [])
 
         if not instance_id:
             raise Error('invalid instance (cluster) id')
@@ -432,6 +451,12 @@ def validate(self) -> None:
             # the cluster/instanced id to the system hostname
             hname = socket.getfqdn()
             vhostname = f'{instance_id}-{hname}'
+        _public_addrs = [
+            ClusterPublicIP.convert(v) for v in cluster_public_addrs
+        ]
+        if _public_addrs:
+            # cache the cephadm networks->devices mapping for later
+            self._network_mapper.load()
 
         self._instance_cfg = Config(
             identity=self._identity,
@@ -447,6 +472,7 @@ def validate(self) -> None:
             vhostname=vhostname,
             cluster_meta_uri=cluster_meta_uri,
             cluster_lock_uri=cluster_lock_uri,
+            cluster_public_addrs=_public_addrs,
         )
         if self._rank_info:
             (
@@ -674,7 +700,54 @@ def _write_ctdb_stub_config(self, path: pathlib.Path) -> None:
                 'recovery_lock': f'!{reclock_cmd}',
                 'cluster_meta_uri': self._cfg.cluster_meta_uri,
                 'nodes_cmd': nodes_cmd,
+                'public_addresses': self._network_mapper.for_sambacc(
+                    self._cfg
+                ),
             },
         }
         with file_utils.write_new(path) as fh:
             json.dump(stub_config, fh)
+
+
+class _NetworkMapper:
+    """Helper class that maps between cephadm-friendly address-networks
+    groupings to ctdb-friendly address-device groupings.
+    """
+
+    def __init__(self, ctx: CephadmContext):
+        self._ctx = ctx
+        self._networks: Dict = {}
+
+    def load(self) -> None:
+        logger.debug('fetching networks')
+        self._networks = list_networks(self._ctx)
+
+    def _convert(self, addr: ClusterPublicIP) -> ClusterPublicIP:
+        devs = []
+        for net in addr.destinations:
+            if net not in self._networks:
+                # ignore mappings that cant exist on this host
+                logger.warning(
+                    'destination network %r not found in %r',
+                    net,
+                    self._networks.keys(),
+                )
+                continue
+            for dev in self._networks[net]:
+                logger.debug(
+                    'adding device %s from network %r for public ip %s',
+                    dev,
+                    net,
+                    addr.address,
+                )
+                devs.append(dev)
+        return ClusterPublicIP(addr.address, devs)
+
+    def for_sambacc(self, cfg: Config) -> List[Dict[str, Any]]:
+        if not cfg.cluster_public_addrs:
+            return []
+        addrs = (self._convert(a) for a in (cfg.cluster_public_addrs or []))
+        return [
+            {'address': a.address, 'interfaces': a.destinations}
+            for a in addrs
+        ]

src/pybind/mgr/cephadm/services/smb.py

@@ -70,6 +70,9 @@ def generate_config(
             config_blobs['cluster_meta_uri'] = smb_spec.cluster_meta_uri
         if smb_spec.cluster_lock_uri:
             config_blobs['cluster_lock_uri'] = smb_spec.cluster_lock_uri
+        cluster_public_addrs = smb_spec.strict_cluster_ip_specs()
+        if cluster_public_addrs:
+            config_blobs['cluster_public_addrs'] = cluster_public_addrs
         ceph_users = smb_spec.include_ceph_users or []
         config_blobs.update(
             self._ceph_config_and_keyring_for(

src/pybind/mgr/smb/handler.py

@@ -1205,6 +1205,7 @@ def _generate_smb_service_spec(
         user_sources=user_sources,
         custom_dns=cluster.custom_dns,
         include_ceph_users=user_entities,
+        cluster_public_addrs=cluster.service_spec_public_addrs(),
     )