diff --git a/docs/siteconfig.rst b/docs/siteconfig.rst index effb8219c..6280ef296 100644 --- a/docs/siteconfig.rst +++ b/docs/siteconfig.rst @@ -255,6 +255,17 @@ Here is a sample configuration with many of the options set and documented:: endpoint: http://head.ses.suse.de:5000/ machine_types: ['type1', 'type2', 'type3'] + # Define a list of ssh tunnels for a various group of test nodes. + # Notice: provided domain names for the nodes must be resolvable + # in your network and jump host (bastion) must be accessible. + tunnel: + - hosts: ['example1.domain', 'example2.domain', 'example3.domain'] + bastion: + host: ssh_host_name # must be resolvable and reachable + user: ssh_user_name # (optional) + port: ssh_port # (optional) + identity: ~/.ssh/id_ed25519 # (optional) + # Do not allow more than that many jobs in a single run by default. # To disable this check use 0. job_threshold: 500 diff --git a/setup.cfg b/setup.cfg index ce8979c8e..d7b147cde 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,6 +56,7 @@ install_requires = python-dateutil requests>2.13.0 sentry-sdk + sshtunnel types-psutil urllib3>=1.25.4,<1.27 # For botocore scripts = diff --git a/teuthology/orchestra/connection.py b/teuthology/orchestra/connection.py index 1772d37b5..9a6443dc2 100644 --- a/teuthology/orchestra/connection.py +++ b/teuthology/orchestra/connection.py @@ -8,6 +8,7 @@ from teuthology.config import config from teuthology.contextutil import safe_while from paramiko.hostkeys import HostKeyEntry +import sshtunnel log = logging.getLogger(__name__) @@ -50,7 +51,20 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60, :param retry: Whether or not to retry failed connection attempts (eventually giving up if none succeed). Default is True :param key_filename: Optionally override which private key to use. + :return: ssh connection. + + The connection is going to be established via tunnel if corresponding options + are provided in teuthology configuration file. For example: + + tunnel: + - hosts: ['hostname1.domain', 'hostname2.domain'] + bastion: + host: ssh_host_name + user: ssh_user_name + port: 22 + identity: ~/.ssh/id_ed25519 + """ user, host = split_user(user_at_host) if _SSHClient is None: @@ -79,6 +93,28 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60, timeout=timeout ) + if config.tunnel: + for tunnel in config.tunnel: + if host in tunnel.get('hosts'): + bastion = tunnel.get('bastion') + if not bastion: + log.error("The 'tunnel' config must include 'bastion' entry") + continue + bastion_host = bastion.get('host') + server = sshtunnel.SSHTunnelForwarder( + bastion_host, + ssh_username=bastion.get('user', None), + ssh_password=bastion.get('word', None), + ssh_pkey=bastion.get('identity'), + remote_bind_address=(host, 22)) + log.info(f'Starting tunnel to {bastion_host} for host {host}') + server.start() + local_port = server.local_bind_port + log.debug(f"Local port for host {host} is {local_port}") + connect_args['hostname'] = '127.0.0.1' + connect_args['port'] = local_port + break + key_filename = key_filename or config.ssh_key ssh_config_path = config.ssh_config_path or "~/.ssh/config" ssh_config_path = os.path.expanduser(ssh_config_path)