diff --git a/changelog/40943.added.md b/changelog/40943.added.md new file mode 100644 index 000000000000..f8f5f8199ffa --- /dev/null +++ b/changelog/40943.added.md @@ -0,0 +1 @@ +Allowed publishing to regular minions from the SSH wrapper diff --git a/changelog/65645.added.md b/changelog/65645.added.md new file mode 100644 index 000000000000..eeddb0f332aa --- /dev/null +++ b/changelog/65645.added.md @@ -0,0 +1 @@ +Allowed accessing the regular mine from the SSH wrapper diff --git a/salt/client/ssh/wrapper/mine.py b/salt/client/ssh/wrapper/mine.py index 1817058eea8a..2eb789e57d88 100644 --- a/salt/client/ssh/wrapper/mine.py +++ b/salt/client/ssh/wrapper/mine.py @@ -2,26 +2,35 @@ Wrapper function for mine operations for salt-ssh .. versionadded:: 2015.5.0 +.. versionchanged:: 3007.0 + + In addition to mine returns from roster targets, this wrapper now supports + accessing the regular mine as well. """ import copy import salt.client.ssh +import salt.daemons.masterapi -def get(tgt, fun, tgt_type="glob", roster="flat"): +def get( + tgt, fun, tgt_type="glob", roster="flat", ssh_minions=True, regular_minions=False +): """ Get data from the mine based on the target, function and tgt_type - This will actually run the function on all targeted minions (like + This will actually run the function on all targeted SSH minions (like publish.publish), as salt-ssh clients can't update the mine themselves. We will look for mine_functions in the roster, pillar, and master config, - in that order, looking for a match for the defined function + in that order, looking for a match for the defined function. Targets can be matched based on any standard matching system that can be - matched on the defined roster (in salt-ssh) via these keywords:: + matched on the defined roster (in salt-ssh). + + Regular mine data will be fetched as usual and can be targeted as usual. CLI Example: @@ -30,30 +39,67 @@ def get(tgt, fun, tgt_type="glob", roster="flat"): salt-ssh '*' mine.get '*' network.interfaces salt-ssh '*' mine.get 'myminion' network.interfaces roster=flat salt-ssh '*' mine.get '192.168.5.0' network.ipaddrs roster=scan + salt-ssh myminion mine.get '*' network.interfaces ssh_minions=False regular_minions=True + salt-ssh myminion mine.get '*' network.interfaces ssh_minions=True regular_minions=True + + tgt + Target whose mine data to get. + + fun + Function to get the mine data of. You can specify multiple functions + to retrieve using either a list or a comma-separated string of functions. + + tgt_type + Target type to use with ``tgt``. Defaults to ``glob``. + See :ref:`targeting` for more information for regular minion targets, above + for SSH ones. + + roster + The roster module to use. Defaults to ``flat``. + + ssh_minions + .. versionadded:: 3007.0 + Target minions from the roster. Defaults to true. + + regular_minions + .. versionadded:: 3007.0 + Target regular minions of the master running salt-ssh. Defaults to false. """ - # Set up opts for the SSH object - opts = copy.deepcopy(__context__["master_opts"]) - minopts = copy.deepcopy(__opts__) - opts.update(minopts) - if roster: - opts["roster"] = roster - opts["argv"] = [fun] - opts["selected_target_option"] = tgt_type - opts["tgt"] = tgt - opts["arg"] = [] - - # Create the SSH object to handle the actual call - ssh = salt.client.ssh.SSH(opts) - - # Run salt-ssh to get the minion returns rets = {} - for ret in ssh.run_iter(mine=True): + if regular_minions: + masterapi = salt.daemons.masterapi.RemoteFuncs(__context__["master_opts"]) + load = { + "id": __opts__["id"], + "fun": fun, + "tgt": tgt, + "tgt_type": tgt_type, + } + ret = masterapi._mine_get(load) rets.update(ret) - cret = {} - for host in rets: - if "return" in rets[host]: - cret[host] = rets[host]["return"] - else: - cret[host] = rets[host] - return cret + if ssh_minions: + # Set up opts for the SSH object + opts = copy.deepcopy(__context__["master_opts"]) + minopts = copy.deepcopy(__opts__) + opts.update(minopts) + if roster: + opts["roster"] = roster + opts["argv"] = [fun] + opts["selected_target_option"] = tgt_type + opts["tgt"] = tgt + opts["arg"] = [] + + # Create the SSH object to handle the actual call + ssh = salt.client.ssh.SSH(opts) + + # Run salt-ssh to get the minion returns + mrets = {} + for ret in ssh.run_iter(mine=True): + mrets.update(ret) + + for host in mrets: + if "return" in mrets[host]: + rets[host] = mrets[host]["return"] + else: + rets[host] = mrets[host] + return rets diff --git a/salt/client/ssh/wrapper/publish.py b/salt/client/ssh/wrapper/publish.py index 0b35312ffb87..7d1e378db30f 100644 --- a/salt/client/ssh/wrapper/publish.py +++ b/salt/client/ssh/wrapper/publish.py @@ -7,18 +7,39 @@ salt-ssh calls and return the data from them. No access control is needed because calls cannot originate from the minions. + +.. versionchanged:: 3007.0 + + In addition to SSH minions, this module can now also target regular ones. """ import copy import logging +import time import salt.client.ssh +import salt.daemons.masterapi import salt.runner import salt.utils.args +import salt.utils.json log = logging.getLogger(__name__) +def _parse_args(arg): + """ + yamlify `arg` and ensure its outermost datatype is a list + """ + yaml_args = salt.utils.args.yamlify_arg(arg) + + if yaml_args is None: + return [] + elif not isinstance(yaml_args, list): + return [yaml_args] + else: + return yaml_args + + def _publish( tgt, fun, @@ -65,9 +86,13 @@ def _publish( if arg is None: arg = [] elif not isinstance(arg, list): - arg = [salt.utils.args.yamlify_arg(arg)] + # yamlify_arg does not operate on non-strings, which we need to JSON-encode + arg = [salt.utils.json.dumps(salt.utils.args.yamlify_arg(arg))] else: - arg = [salt.utils.args.yamlify_arg(x) for x in arg] + arg = [ + salt.utils.json.dumps(y) + for y in (salt.utils.args.yamlify_arg(x) for x in arg) + ] if len(arg) == 1 and arg[0] is None: arg = [] @@ -100,29 +125,130 @@ def _publish( else: cret[host] = rets[host] return cret - else: - return rets + for host in rets: + if "return" in rets[host]: + # The regular publish return just contains `ret`, + # at least make it accessible like this as well + rets[host]["ret"] = rets[host]["return"] + return rets + +def _publish_regular( + tgt, + fun, + arg=None, + tgt_type="glob", + returner="", + timeout=5, + form="clean", + wait=False, +): + if fun.startswith("publish."): + log.info("Cannot publish publish calls. Returning {}") + return {} -def publish(tgt, fun, arg=None, tgt_type="glob", returner="", timeout=5, roster=None): + arg = _parse_args(arg) + masterapi = salt.daemons.masterapi.RemoteFuncs(__context__["master_opts"]) + + log.info("Publishing '%s'", fun) + load = { + "cmd": "minion_pub", + "fun": fun, + "arg": arg, + "tgt": tgt, + "tgt_type": tgt_type, + "ret": returner, + "tmo": timeout, + "form": form, + "id": __opts__["id"], + "no_parse": __opts__.get("no_parse", []), + } + peer_data = masterapi.minion_pub(load) + if not peer_data: + return {} + # CLI args are passed as strings, re-cast to keep time.sleep happy + if wait: + loop_interval = 0.3 + matched_minions = set(peer_data["minions"]) + returned_minions = set() + loop_counter = 0 + while returned_minions ^ matched_minions: + load = { + "cmd": "pub_ret", + "id": __opts__["id"], + "jid": peer_data["jid"], + } + ret = masterapi.pub_ret(load) + returned_minions = set(ret.keys()) + + end_loop = False + if returned_minions >= matched_minions: + end_loop = True + elif (loop_interval * loop_counter) > timeout: + if not returned_minions: + return {} + end_loop = True + + if end_loop: + if form == "clean": + cret = {} + for host in ret: + cret[host] = ret[host]["ret"] + return cret + else: + return ret + loop_counter = loop_counter + 1 + time.sleep(loop_interval) + else: + time.sleep(float(timeout)) + load = { + "cmd": "pub_ret", + "id": __opts__["id"], + "jid": peer_data["jid"], + } + ret = masterapi.pub_ret(load) + if form == "clean": + cret = {} + for host in ret: + cret[host] = ret[host]["ret"] + return cret + else: + return ret + return ret + + +def publish( + tgt, + fun, + arg=None, + tgt_type="glob", + returner="", + timeout=5, + roster=None, + ssh_minions=True, + regular_minions=False, +): """ - Publish a command "from the minion out to other minions". In reality, the + Publish a command from the minion out to other minions. In reality, the minion does not execute this function, it is executed by the master. Thus, no access control is enabled, as minions cannot initiate publishes themselves. - Salt-ssh publishes will default to whichever roster was used for the initiating salt-ssh call, and can be overridden using the ``roster`` - argument + argument. Returners are not currently supported The tgt_type argument is used to pass a target other than a glob into - the execution, the available options are: + the execution, the available options for SSH minions are: - glob - pcre + - nodegroup + - range + + Regular minions support all usual ones. .. versionchanged:: 2017.7.0 The ``expr_form`` argument has been renamed to ``tgt_type``, earlier @@ -161,21 +287,77 @@ def publish(tgt, fun, arg=None, tgt_type="glob", returner="", timeout=5, roster= salt-ssh '*' publish.publish test.kwarg arg="['cheese=spam','spam=cheese']" + tgt + The target specification. + + fun + The execution module to run. + + arg + A list of arguments to pass to the module. + + tgt_type + The matcher to use. Defaults to ``glob``. + + returner + A returner to use. + timeout + Timeout in seconds. Defaults to 5. + + roster + Override the roster for SSH minion targets. Defaults to the one + used for initiating the salt-ssh call. + + ssh_minions + .. versionadded:: 3007.0 + Include SSH minions in the possible targets. Defaults to true. + + regular_minions + .. versionadded:: 3007.0 + Include regular minions in the possible targets. Defaults to false. """ - return _publish( - tgt, - fun, - arg=arg, - tgt_type=tgt_type, - returner=returner, - timeout=timeout, - form="clean", - roster=roster, - ) - - -def full_data(tgt, fun, arg=None, tgt_type="glob", returner="", timeout=5, roster=None): + rets = {} + if regular_minions: + rets.update( + _publish_regular( + tgt, + fun, + arg=arg, + tgt_type=tgt_type, + returner=returner, + timeout=timeout, + form="clean", + wait=True, + ) + ) + if ssh_minions: + rets.update( + _publish( + tgt, + fun, + arg=arg, + tgt_type=tgt_type, + returner=returner, + timeout=timeout, + form="clean", + roster=roster, + ) + ) + return rets + + +def full_data( + tgt, + fun, + arg=None, + tgt_type="glob", + returner="", + timeout=5, + roster=None, + ssh_minions=True, + regular_minions=False, +): """ Return the full data about the publication, this is invoked in the same way as the publish function @@ -197,21 +379,39 @@ def full_data(tgt, fun, arg=None, tgt_type="glob", returner="", timeout=5, roste salt-ssh '*' publish.full_data test.kwarg arg='cheese=spam' """ - return _publish( - tgt, - fun, - arg=arg, - tgt_type=tgt_type, - returner=returner, - timeout=timeout, - form="full", - roster=roster, - ) + rets = {} + if regular_minions: + rets.update( + _publish_regular( + tgt, + fun, + arg=arg, + tgt_type=tgt_type, + returner=returner, + timeout=timeout, + form="full", + wait=True, + ) + ) + if ssh_minions: + rets.update( + _publish( + tgt, + fun, + arg=arg, + tgt_type=tgt_type, + returner=returner, + timeout=timeout, + form="full", + roster=roster, + ) + ) + return rets def runner(fun, arg=None, timeout=5): """ - Execute a runner on the master and return the data from the runnr function + Execute a runner on the master and return the data from the runner function CLI Example: diff --git a/tests/pytests/integration/ssh/test_mine.py b/tests/pytests/integration/ssh/test_mine.py index 7bb5bb10822a..a39760b7652c 100644 --- a/tests/pytests/integration/ssh/test_mine.py +++ b/tests/pytests/integration/ssh/test_mine.py @@ -29,3 +29,28 @@ def test_ssh_mine_get(salt_ssh_cli): assert "localhost" in ret.data assert "args" in ret.data["localhost"] assert ret.data["localhost"]["args"] == ["itworked"] + + +@pytest.mark.parametrize("tgts", (("ssh",), ("regular",), ("ssh", "regular"))) +def test_mine_get(salt_ssh_cli, salt_minion, tgts): + """ + Test mine returns with both regular and SSH minions + """ + if len(tgts) > 1: + tgt = "*" + exp = {"localhost", salt_minion.id} + else: + tgt = "localhost" if "ssh" in tgts else salt_minion.id + exp = {tgt} + ret = salt_ssh_cli.run( + "mine.get", + "*", + "test.ping", + ssh_minions="ssh" in tgts, + regular_minions="regular" in tgts, + ) + assert ret.returncode == 0 + assert ret.data + assert set(ret.data) == exp + for id_ in exp: + assert ret.data[id_] is True diff --git a/tests/pytests/integration/ssh/test_publish.py b/tests/pytests/integration/ssh/test_publish.py new file mode 100644 index 000000000000..4ecfaeb349a0 --- /dev/null +++ b/tests/pytests/integration/ssh/test_publish.py @@ -0,0 +1,106 @@ +import pytest + +pytestmark = [ + pytest.mark.slow_test, + pytest.mark.skip_on_windows(reason="salt-ssh not available on Windows"), +] + + +@pytest.mark.parametrize("tgts", (("ssh",), ("regular",), ("ssh", "regular"))) +def test_publish(salt_ssh_cli, salt_minion, tgts): + if len(tgts) > 1: + tgt = "*" + exp = {"localhost", salt_minion.id} + else: + tgt = "localhost" if "ssh" in tgts else salt_minion.id + exp = {tgt} + ret = salt_ssh_cli.run( + "publish.publish", + tgt, + "test.ping", + ssh_minions="ssh" in tgts, + regular_minions="regular" in tgts, + ) + assert ret.returncode == 0 + assert ret.data + assert set(ret.data) == exp + for id_ in exp: + assert ret.data[id_] is True + + +def test_publish_with_arg(salt_ssh_cli, salt_minion): + ret = salt_ssh_cli.run( + "publish.publish", + "*", + "test.kwarg", + arg=["cheese=spam"], + ssh_minions=True, + regular_minions=True, + ) + assert ret.returncode == 0 + assert ret.data + exp = {salt_minion.id, "localhost"} + assert set(ret.data) == exp + for id_ in exp: + assert ret.data[id_]["cheese"] == "spam" + + +def test_publish_with_yaml_args(salt_ssh_cli, salt_minion): + args = ["saltines, si", "crackers, nein", "cheese, indeed"] + test_args = f'["{args[0]}", "{args[1]}", "{args[2]}"]' + ret = salt_ssh_cli.run( + "publish.publish", + "*", + "test.arg", + arg=test_args, + ssh_minions=True, + regular_minions=True, + ) + assert ret.returncode == 0 + assert ret.data + exp = {salt_minion.id, "localhost"} + assert set(ret.data) == exp + for id_ in exp: + assert ret.data[id_]["args"] == args + + +@pytest.mark.parametrize("tgts", (("ssh",), ("regular",), ("ssh", "regular"))) +def test_full_data(salt_ssh_cli, salt_minion, tgts): + if len(tgts) > 1: + tgt = "*" + exp = {"localhost", salt_minion.id} + else: + tgt = "localhost" if "ssh" in tgts else salt_minion.id + exp = {tgt} + ret = salt_ssh_cli.run( + "publish.full_data", + tgt, + "test.fib", + arg=20, + ssh_minions="ssh" in tgts, + regular_minions="regular" in tgts, + ) + assert ret.returncode == 0 + assert ret.data + assert set(ret.data) == exp + for id_ in exp: + assert "ret" in ret.data[id_] + assert ret.data[id_]["ret"][0] == 6765 + + +def test_full_data_kwarg(salt_ssh_cli, salt_minion): + ret = salt_ssh_cli.run( + "publish.full_data", + "*", + "test.kwarg", + arg=["cheese=spam"], + ssh_minions=True, + regular_minions=True, + ) + assert ret.returncode == 0 + assert ret.data + exp = {"localhost", salt_minion.id} + assert set(ret.data) == exp + for id_ in exp: + assert "ret" in ret.data[id_] + assert ret.data[id_]["ret"]["cheese"] == "spam"