From 040ae9e55ccf01acf5ce9c5daefc0006e4cc15c3 Mon Sep 17 00:00:00 2001 From: Nicholas Hughes Date: Wed, 15 Nov 2023 14:48:45 -0500 Subject: [PATCH] [master] Add wildcard removal for aptpkg (#65221) * fixes saltstack/salt#65220 add wildcard removal for aptpkg * adding functional module tests for wildcard removal * fix functional pkg tests for wildcard * fix functional pkg tests for wildcard * fix functional pkg tests for wildcard * fix functional pkg tests for wildcard * fix functional pkg tests for wildcard * fix functional pkg tests for wildcard * fix functional pkg tests for wildcard * fix functional pkg tests for wildcard * fix functional pkg tests for wildcard * fix functional pkg tests for wildcard * fix functional pkg tests for wildcard * adding pytest marks to skip yum tests on non-el systems * fixing update method for match_wildcard dict --------- Co-authored-by: Megan Wilhite --- changelog/65220.added.md | 1 + salt/modules/aptpkg.py | 4 +- salt/modules/yumpkg.py | 17 +-------- salt/utils/pkg/__init__.py | 31 +++++++++++++++ .../pytests/functional/modules/test_aptpkg.py | 26 ++++++++++++- .../pytests/functional/modules/test_yumpkg.py | 38 ++++++++++++++++--- tests/pytests/unit/utils/test_pkg.py | 30 +++++++++++++++ 7 files changed, 122 insertions(+), 25 deletions(-) create mode 100644 changelog/65220.added.md create mode 100644 tests/pytests/unit/utils/test_pkg.py diff --git a/changelog/65220.added.md b/changelog/65220.added.md new file mode 100644 index 000000000000..6db0a4c4b0a2 --- /dev/null +++ b/changelog/65220.added.md @@ -0,0 +1 @@ +Add ability to remove packages by wildcard via apt execution module diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index 70db64b8405d..06cfad43e84d 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -1057,9 +1057,9 @@ def _uninstall(action="remove", name=None, pkgs=None, **kwargs): old = list_pkgs() old_removed = list_pkgs(removed=True) - targets = [x for x in pkg_params if x in old] + targets = salt.utils.pkg.match_wildcard(old, pkg_params) if action == "purge": - targets.extend([x for x in pkg_params if x in old_removed]) + targets.update(salt.utils.pkg.match_wildcard(old_removed, pkg_params)) if not targets: return {} cmd = ["apt-get", "-q", "-y", action] diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 4262057f55ff..e3e4a689b048 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -2155,22 +2155,7 @@ def remove(name=None, pkgs=None, **kwargs): # pylint: disable=W0613 old = list_pkgs() targets = [] - # Loop through pkg_params looking for any - # which contains a wildcard and get the - # real package names from the packages - # which are currently installed. - pkg_matches = {} - for pkg_param in list(pkg_params): - if "*" in pkg_param: - pkg_matches = { - x: pkg_params[pkg_param] for x in old if fnmatch.fnmatch(x, pkg_param) - } - - # Remove previous pkg_param - pkg_params.pop(pkg_param) - - # Update pkg_params with the matches - pkg_params.update(pkg_matches) + pkg_params = salt.utils.pkg.match_wildcard(old, pkg_params) for target in pkg_params: if target not in old: diff --git a/salt/utils/pkg/__init__.py b/salt/utils/pkg/__init__.py index 9a11e492a046..7092e7a5c774 100644 --- a/salt/utils/pkg/__init__.py +++ b/salt/utils/pkg/__init__.py @@ -3,6 +3,7 @@ """ import errno +import fnmatch import logging import os import re @@ -102,3 +103,33 @@ def check_bundled(): if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): return True return False + + +def match_wildcard(current_pkgs, pkg_params): + """ + Loop through pkg_params looking for any which contains a wildcard and get + the real package names from the packages which are currently installed. + + current_pkgs + List of currently installed packages as output by ``list_pkgs`` + + pkg_params + List of packages as processed by ``pkg_resource.parse_targets`` + """ + pkg_matches = {} + + for pkg_param in list(pkg_params): + if "*" in pkg_param: + pkg_matches = { + pkg: pkg_params[pkg_param] + for pkg in current_pkgs + if fnmatch.fnmatch(pkg, pkg_param) + } + + # Remove previous pkg_param + pkg_params.pop(pkg_param) + + # Update pkg_params with the matches + pkg_params.update(pkg_matches) + + return pkg_params diff --git a/tests/pytests/functional/modules/test_aptpkg.py b/tests/pytests/functional/modules/test_aptpkg.py index 7d4933875d41..a9063e48b8b8 100644 --- a/tests/pytests/functional/modules/test_aptpkg.py +++ b/tests/pytests/functional/modules/test_aptpkg.py @@ -11,6 +11,7 @@ import salt.modules.cp as cp import salt.modules.file as file import salt.modules.gpg as gpg +import salt.modules.pkg_resource as pkg_resource import salt.utils.files import salt.utils.stringutils from tests.support.mock import Mock, patch @@ -52,7 +53,9 @@ def get_key_file(request, state_tree, functional_files_dir): @pytest.fixture -def configure_loader_modules(minion_opts): +def configure_loader_modules(minion_opts, grains): + osarch = cmd.run("dpkg --print-architecture").strip() + grains.update({"osarch": osarch}) return { aptpkg: { "__salt__": { @@ -63,8 +66,15 @@ def configure_loader_modules(minion_opts): "file.grep": file.grep, "cp.cache_file": cp.cache_file, "config.get": config.get, + "cmd.run_stdout": cmd.run_stdout, + "pkg_resource.add_pkg": pkg_resource.add_pkg, + "pkg_resource.format_pkg_list": pkg_resource.format_pkg_list, + "pkg_resource.parse_targets": pkg_resource.parse_targets, + "pkg_resource.sort_pkglist": pkg_resource.sort_pkglist, + "pkg_resource.stringify": pkg_resource.stringify, }, "__opts__": minion_opts, + "__grains__": grains, }, file: { "__salt__": {"cmd.run_all": cmd.run_all}, @@ -81,6 +91,9 @@ def configure_loader_modules(minion_opts): config: { "__opts__": minion_opts, }, + pkg_resource: { + "__grains__": grains, + }, } @@ -374,3 +387,14 @@ def test_add_del_repo_key(get_key_file, aptkey): assert not keyfile.is_file() query_key = aptpkg.get_repo_keys(aptkey=aptkey) assert "0E08A149DE57BFBE" not in query_key + + +@pytest.mark.destructive_test +@pytest.mark.skip_if_not_root +def test_aptpkg_remove_wildcard(): + aptpkg.install(pkgs=["nginx-doc", "nginx-light"]) + ret = aptpkg.remove(name="nginx-*") + assert not ret["nginx-light"]["new"] + assert ret["nginx-light"]["old"] + assert not ret["nginx-doc"]["new"] + assert ret["nginx-doc"]["old"] diff --git a/tests/pytests/functional/modules/test_yumpkg.py b/tests/pytests/functional/modules/test_yumpkg.py index 36b357a61705..88dcf0ee56b4 100644 --- a/tests/pytests/functional/modules/test_yumpkg.py +++ b/tests/pytests/functional/modules/test_yumpkg.py @@ -1,34 +1,49 @@ import pytest import salt.modules.cmdmod +import salt.modules.config import salt.modules.pkg_resource import salt.modules.yumpkg import salt.utils.pkg.rpm +pytestmark = [ + pytest.mark.skip_if_binaries_missing("rpm", "yum"), + pytest.mark.slow_test, +] + @pytest.fixture -def configure_loader_modules(minion_opts): +def configure_loader_modules(minion_opts, grains): + grains.update({"osarch": salt.utils.pkg.rpm.get_osarch()}) return { + salt.modules.config: { + "__grains__": grains, + }, + salt.modules.pkg_resource: { + "__grains__": grains, + }, salt.modules.yumpkg: { "__salt__": { "cmd.run": salt.modules.cmdmod.run, + "cmd.run_all": salt.modules.cmdmod.run_all, + "cmd.run_stdout": salt.modules.cmdmod.run_stdout, + "config.get": salt.modules.config.get, "pkg_resource.add_pkg": salt.modules.pkg_resource.add_pkg, "pkg_resource.format_pkg_list": salt.modules.pkg_resource.format_pkg_list, + "pkg_resource.parse_targets": salt.modules.pkg_resource.parse_targets, + "pkg_resource.sort_pkglist": salt.modules.pkg_resource.sort_pkglist, }, - "__grains__": {"osarch": salt.utils.pkg.rpm.get_osarch()}, + "__opts__": minion_opts, + "__grains__": grains, }, } -@pytest.mark.slow_test def test_yum_list_pkgs(grains): """ compare the output of rpm -qa vs the return of yumpkg.list_pkgs, make sure that any changes to ympkg.list_pkgs still returns. """ - - if grains["os_family"] != "RedHat": - pytest.skip("Skip if not RedHat") cmd = [ "rpm", "-qa", @@ -39,3 +54,14 @@ def test_yum_list_pkgs(grains): listed_pkgs = salt.modules.yumpkg.list_pkgs() for line in known_pkgs.splitlines(): assert any(line in d for d in listed_pkgs) + + +@pytest.mark.destructive_test +@pytest.mark.skip_if_not_root +def test_yumpkg_remove_wildcard(): + salt.modules.yumpkg.install(pkgs=["httpd-devel", "httpd-tools"]) + ret = salt.modules.yumpkg.remove(name="httpd-*") + assert not ret["httpd-devel"]["new"] + assert ret["httpd-devel"]["old"] + assert not ret["httpd-tools"]["new"] + assert ret["httpd-tools"]["old"] diff --git a/tests/pytests/unit/utils/test_pkg.py b/tests/pytests/unit/utils/test_pkg.py new file mode 100644 index 000000000000..d35432909091 --- /dev/null +++ b/tests/pytests/unit/utils/test_pkg.py @@ -0,0 +1,30 @@ +import pytest + +import salt.utils.pkg + +CURRENT_PKGS = { + "acl": "2.2.53-4", + "adduser": "3.118", + "apparmor": "2.13.2-10", + "apt": "1.8.2.3", + "apt-listchanges": "3.19", + "apt-transport-https": "1.8.2.3", + "apt-utils": "1.8.2.3", + "base-files": "10.3+deb10u13", + "base-passwd": "3.5.46", + "bash": "5.0-4", + "bash-completion": "1:2.8-6", +} + + +@pytest.mark.parametrize( + "current_pkgs,pkg_params,expected", + [ + [CURRENT_PKGS, {"apt": ""}, {"apt": ""}], + [CURRENT_PKGS, {"foo": ""}, {"foo": ""}], + [CURRENT_PKGS, {"bash*": ""}, {"bash": "", "bash-completion": ""}], + ], +) +def test_match_wildcard(current_pkgs, pkg_params, expected): + result = salt.utils.pkg.match_wildcard(current_pkgs, pkg_params) + assert result == expected