From ede5bbe25a3f7aa762fda5b9fa93237334ac6cd6 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 21 Nov 2024 16:06:55 -0500 Subject: [PATCH 01/18] - add new entry points - add logic to check what version of the project is running #rename temp_var, remove old code and add validation check --- awx/main/models/credential.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 1cdf11d135b8..215a08da72d9 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -2,6 +2,7 @@ # All Rights Reserved. from contextlib import nullcontext import functools + import inspect import logging from importlib.metadata import entry_points @@ -45,6 +46,9 @@ ) from awx.main.models import Team, Organization from awx.main.utils import encrypt_field +from awx_plugins.credentials import injectors as builtin_injectors +from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name + # DAB from ansible_base.resource_registry.tasks.sync import get_resource_server_client @@ -645,7 +649,21 @@ def get_absolute_url(self, request=None): return reverse(view_name, kwargs={'pk': self.pk}, request=request) -from awx_plugins.credentials.plugins import * # noqa +awx_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials')} +supported_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials.supported')} +entry_points = awx_entry_points if detect_server_product_name() == 'AWX' else {**awx_entry_points, **supported_entry_points} + +for ns, ep in entry_points.items(): + cred_plugin = ep.load() + if not hasattr(cred_plugin, 'inputs'): + setattr(cred_plugin, 'inputs', {}) + if not hasattr(cred_plugin, 'injectors'): + setattr(cred_plugin, 'injectors', {}) + if ns in ManagedCredentialType.registry: + raise ValueError( + 'a ManagedCredentialType with namespace={} is already defined in {}'.format(ns, inspect.getsourcefile(ManagedCredentialType.registry[ns].__class__)) + ) + ManagedCredentialType.registry[ns] = cred_plugin for ns, plugin in credential_plugins.items(): CredentialType.load_plugin(ns, plugin) From 4864a89984f4aa64c0d165c1e8336e4000d9771d Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Mon, 16 Dec 2024 14:21:37 -0500 Subject: [PATCH 02/18] remove former discovery method --- awx/main/models/credential.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 215a08da72d9..54dfbf7be27e 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -556,19 +556,6 @@ def inject_credential(self, credential, env, safe_env, args, private_data_dir): class ManagedCredentialType(SimpleNamespace): registry = {} - def __init__(self, namespace, **kwargs): - for k in ('inputs', 'injectors'): - if k not in kwargs: - kwargs[k] = {} - super(ManagedCredentialType, self).__init__(namespace=namespace, **kwargs) - if namespace in ManagedCredentialType.registry: - raise ValueError( - 'a ManagedCredentialType with namespace={} is already defined in {}'.format( - namespace, inspect.getsourcefile(ManagedCredentialType.registry[namespace].__class__) - ) - ) - ManagedCredentialType.registry[namespace] = self - def get_creation_params(self): return dict( namespace=self.namespace, From 6e49cbb18f6509b480efcd8a8184f03462410f96 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Tue, 17 Dec 2024 13:25:14 -0500 Subject: [PATCH 03/18] update custom_injectors and remove unused import --- awx/main/models/credential.py | 1 - 1 file changed, 1 deletion(-) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 54dfbf7be27e..26845d161501 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -46,7 +46,6 @@ ) from awx.main.models import Team, Organization from awx.main.utils import encrypt_field -from awx_plugins.credentials import injectors as builtin_injectors from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name From 8f014399fe03435f109b7e1e846ac3c017dd9e20 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 19 Dec 2024 14:55:28 -0500 Subject: [PATCH 04/18] fix how we load external creds --- awx/main/models/credential.py | 172 ++++++++++++++++++++++++++++++---- 1 file changed, 152 insertions(+), 20 deletions(-) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 26845d161501..676502608327 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -57,7 +57,6 @@ __all__ = ['Credential', 'CredentialType', 'CredentialInputSource', 'build_safe_env'] logger = logging.getLogger('awx.main.models.credential') -credential_plugins = {entry_point.name: entry_point.load() for entry_point in entry_points(group='awx_plugins.credentials')} HIDDEN_PASSWORD = '**********' @@ -475,7 +474,7 @@ def default_for_field(self, field_id): @classproperty def defaults(cls): - return dict((k, functools.partial(v.create)) for k, v in ManagedCredentialType.registry.items()) + return dict((k, functools.partial(CredentialTypeHelper.create, v)) for k, v in ManagedCredentialType.registry.items()) @classmethod def _get_credential_type_class(cls, apps: Apps = None, app_config: AppConfig = None): @@ -510,7 +509,7 @@ def _setup_tower_managed_defaults(cls, apps: Apps = None, app_config: AppConfig existing.save() continue logger.debug(_("adding %s credential type" % default.name)) - params = default.get_creation_params() + params = CredentialTypeHelper.get_creation_params(default) if 'managed' not in [f.name for f in ct_class._meta.get_fields()]: params['managed_by_tower'] = params.pop('managed') params['created'] = params['modified'] = now() # CreatedModifiedModel service @@ -544,33 +543,161 @@ def setup_tower_managed_defaults(cls, apps: Apps = None, app_config: AppConfig = @classmethod def load_plugin(cls, ns, plugin): # TODO: User "side-loaded" credential custom_injectors isn't supported - ManagedCredentialType(namespace=ns, name=plugin.name, kind='external', inputs=plugin.inputs) + ManagedCredentialType.registry[ns] = ManagedCredentialType(namespace=ns, name=plugin.name, kind='external', inputs=plugin.inputs, injectors={}) def inject_credential(self, credential, env, safe_env, args, private_data_dir): - from awx_plugins.interfaces._temporary_private_inject_api import inject_credential + """ + Inject credential data into the environment variables and arguments + passed to `ansible-playbook` + + :param credential: a :class:`awx.main.models.Credential` instance + :param env: a dictionary of environment variables used in + the `ansible-playbook` call. This method adds + additional environment variables based on + custom `env` injectors defined on this + CredentialType. + :param safe_env: a dictionary of environment variables stored + in the database for the job run + (`UnifiedJob.job_env`); secret values should + be stripped + :param args: a list of arguments passed to + `ansible-playbook` in the style of + `subprocess.call(args)`. This method appends + additional arguments based on custom + `extra_vars` injectors defined on this + CredentialType. + :param private_data_dir: a temporary directory to store files generated + by `file` injectors (like config files or key + files) + """ + if not self.injectors: + if self.managed and credential.credential_type.custom_injectors: + injected_env = {} + credential.credential_type.custom_injectors(credential, injected_env, private_data_dir) + env.update(injected_env) + safe_env.update(build_safe_env(injected_env)) + return - inject_credential(self, credential, env, safe_env, args, private_data_dir) + class TowerNamespace: + pass + tower_namespace = TowerNamespace() -class ManagedCredentialType(SimpleNamespace): - registry = {} + # maintain a normal namespace for building the ansible-playbook arguments (env and args) + namespace = {'tower': tower_namespace} - def get_creation_params(self): + # maintain a sanitized namespace for building the DB-stored arguments (safe_env) + safe_namespace = {'tower': tower_namespace} + + # build a normal namespace with secret values decrypted (for + # ansible-playbook) and a safe namespace with secret values hidden (for + # DB storage) + for field_name in credential.get_input_keys(): + value = credential.get_input(field_name) + + if type(value) is bool: + # boolean values can't be secret/encrypted/external + safe_namespace[field_name] = namespace[field_name] = value + continue + + if field_name in self.secret_fields: + safe_namespace[field_name] = '**********' + elif len(value): + safe_namespace[field_name] = value + if len(value): + namespace[field_name] = value + + for field in self.inputs.get('fields', []): + # default missing boolean fields to False + if field['type'] == 'boolean' and field['id'] not in credential.inputs.keys(): + namespace[field['id']] = safe_namespace[field['id']] = False + # make sure private keys end with a \n + if field.get('format') == 'ssh_private_key': + if field['id'] in namespace and not namespace[field['id']].endswith('\n'): + namespace[field['id']] += '\n' + + file_tmpls = self.injectors.get('file', {}) + # If any file templates are provided, render the files and update the + # special `tower` template namespace so the filename can be + # referenced in other injectors + + sandbox_env = sandbox.ImmutableSandboxedEnvironment() + + for file_label, file_tmpl in file_tmpls.items(): + data = sandbox_env.from_string(file_tmpl).render(**namespace) + _, path = tempfile.mkstemp(dir=os.path.join(private_data_dir, 'env')) + with open(path, 'w') as f: + f.write(data) + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + container_path = get_incontainer_path(path, private_data_dir) + + # determine if filename indicates single file or many + if file_label.find('.') == -1: + tower_namespace.filename = container_path + else: + if not hasattr(tower_namespace, 'filename'): + tower_namespace.filename = TowerNamespace() + file_label = file_label.split('.')[1] + setattr(tower_namespace.filename, file_label, container_path) + + injector_field = self._meta.get_field('injectors') + for env_var, tmpl in self.injectors.get('env', {}).items(): + try: + injector_field.validate_env_var_allowed(env_var) + except ValidationError as e: + logger.error('Ignoring prohibited env var {}, reason: {}'.format(env_var, e)) + continue + env[env_var] = sandbox_env.from_string(tmpl).render(**namespace) + safe_env[env_var] = sandbox_env.from_string(tmpl).render(**safe_namespace) + + if 'INVENTORY_UPDATE_ID' not in env: + # awx-manage inventory_update does not support extra_vars via -e + def build_extra_vars(node): + if isinstance(node, dict): + return {build_extra_vars(k): build_extra_vars(v) for k, v in node.items()} + elif isinstance(node, list): + return [build_extra_vars(x) for x in node] + else: + return sandbox_env.from_string(node).render(**namespace) + + def build_extra_vars_file(vars, private_dir): + handle, path = tempfile.mkstemp(dir=os.path.join(private_dir, 'env')) + f = os.fdopen(handle, 'w') + f.write(safe_dump(vars)) + f.close() + os.chmod(path, stat.S_IRUSR) + return path + + extra_vars = build_extra_vars(self.injectors.get('extra_vars', {})) + if extra_vars: + path = build_extra_vars_file(extra_vars, private_data_dir) + container_path = get_incontainer_path(path, private_data_dir) + args.extend(['-e', '@%s' % container_path]) + + +class CredentialTypeHelper: + @classmethod + def get_creation_params(cls, cred_type): return dict( - namespace=self.namespace, - kind=self.kind, - name=self.name, + namespace=cred_type.namespace, + kind=cred_type.kind, + name=cred_type.name, managed=True, - inputs=self.inputs, - injectors=self.injectors, + inputs=cred_type.inputs, + injectors=cred_type.injectors, ) - def create(self): - res = CredentialType(**self.get_creation_params()) - res.custom_injectors = getattr(self, 'custom_injectors', None) + @classmethod + def create(cls, cred_type): + res = CredentialType(**CredentialTypeHelper.get_creation_params(cred_type)) + res.custom_injectors = cred_type.custom_injectors return res +class ManagedCredentialType(SimpleNamespace): + registry = {} + + class CredentialInputSource(PrimordialModel): class Meta: app_label = 'main' @@ -637,9 +764,9 @@ def get_absolute_url(self, request=None): awx_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials')} supported_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials.supported')} -entry_points = awx_entry_points if detect_server_product_name() == 'AWX' else {**awx_entry_points, **supported_entry_points} +plugin_entry_points = awx_entry_points if detect_server_product_name() == 'AWX' else {**awx_entry_points, **supported_entry_points} -for ns, ep in entry_points.items(): +for ns, ep in plugin_entry_points.items(): cred_plugin = ep.load() if not hasattr(cred_plugin, 'inputs'): setattr(cred_plugin, 'inputs', {}) @@ -651,5 +778,10 @@ def get_absolute_url(self, request=None): ) ManagedCredentialType.registry[ns] = cred_plugin -for ns, plugin in credential_plugins.items(): +credential_plugins = {ep.name: ep for ep in entry_points(group='awx_plugins.credentials')} +if detect_server_product_name() == 'AWX': + credential_plugins = {} + +for ns, ep in credential_plugins.items(): + plugin = ep.load() CredentialType.load_plugin(ns, plugin) From 2acada26a6775467d764ef20dc432c600ee9e81b Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 19 Dec 2024 11:56:11 -0500 Subject: [PATCH 05/18] linter updates --- awx/main/models/credential.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 676502608327..7b3b8b9a4183 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -675,6 +675,7 @@ def build_extra_vars_file(vars, private_dir): args.extend(['-e', '@%s' % container_path]) + class CredentialTypeHelper: @classmethod def get_creation_params(cls, cred_type): From 1301c6bc1cc23482a7dfb6f3541276a58944f067 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 19 Dec 2024 15:24:55 -0500 Subject: [PATCH 06/18] linter fixes --- awx/main/models/credential.py | 1 - 1 file changed, 1 deletion(-) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 7b3b8b9a4183..676502608327 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -675,7 +675,6 @@ def build_extra_vars_file(vars, private_dir): args.extend(['-e', '@%s' % container_path]) - class CredentialTypeHelper: @classmethod def get_creation_params(cls, cred_type): From b1a2aaee29395c7f4f64e8ac1253a74ec1d59b2b Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 19 Dec 2024 15:50:37 -0500 Subject: [PATCH 07/18] remove stale code to match devel --- awx/main/models/credential.py | 130 +--------------------------------- 1 file changed, 3 insertions(+), 127 deletions(-) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 676502608327..c8cbb96db967 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -546,133 +546,9 @@ def load_plugin(cls, ns, plugin): ManagedCredentialType.registry[ns] = ManagedCredentialType(namespace=ns, name=plugin.name, kind='external', inputs=plugin.inputs, injectors={}) def inject_credential(self, credential, env, safe_env, args, private_data_dir): - """ - Inject credential data into the environment variables and arguments - passed to `ansible-playbook` - - :param credential: a :class:`awx.main.models.Credential` instance - :param env: a dictionary of environment variables used in - the `ansible-playbook` call. This method adds - additional environment variables based on - custom `env` injectors defined on this - CredentialType. - :param safe_env: a dictionary of environment variables stored - in the database for the job run - (`UnifiedJob.job_env`); secret values should - be stripped - :param args: a list of arguments passed to - `ansible-playbook` in the style of - `subprocess.call(args)`. This method appends - additional arguments based on custom - `extra_vars` injectors defined on this - CredentialType. - :param private_data_dir: a temporary directory to store files generated - by `file` injectors (like config files or key - files) - """ - if not self.injectors: - if self.managed and credential.credential_type.custom_injectors: - injected_env = {} - credential.credential_type.custom_injectors(credential, injected_env, private_data_dir) - env.update(injected_env) - safe_env.update(build_safe_env(injected_env)) - return - - class TowerNamespace: - pass - - tower_namespace = TowerNamespace() - - # maintain a normal namespace for building the ansible-playbook arguments (env and args) - namespace = {'tower': tower_namespace} - - # maintain a sanitized namespace for building the DB-stored arguments (safe_env) - safe_namespace = {'tower': tower_namespace} - - # build a normal namespace with secret values decrypted (for - # ansible-playbook) and a safe namespace with secret values hidden (for - # DB storage) - for field_name in credential.get_input_keys(): - value = credential.get_input(field_name) + from awx_plugins.interfaces._temporary_private_inject_api import inject_credential - if type(value) is bool: - # boolean values can't be secret/encrypted/external - safe_namespace[field_name] = namespace[field_name] = value - continue - - if field_name in self.secret_fields: - safe_namespace[field_name] = '**********' - elif len(value): - safe_namespace[field_name] = value - if len(value): - namespace[field_name] = value - - for field in self.inputs.get('fields', []): - # default missing boolean fields to False - if field['type'] == 'boolean' and field['id'] not in credential.inputs.keys(): - namespace[field['id']] = safe_namespace[field['id']] = False - # make sure private keys end with a \n - if field.get('format') == 'ssh_private_key': - if field['id'] in namespace and not namespace[field['id']].endswith('\n'): - namespace[field['id']] += '\n' - - file_tmpls = self.injectors.get('file', {}) - # If any file templates are provided, render the files and update the - # special `tower` template namespace so the filename can be - # referenced in other injectors - - sandbox_env = sandbox.ImmutableSandboxedEnvironment() - - for file_label, file_tmpl in file_tmpls.items(): - data = sandbox_env.from_string(file_tmpl).render(**namespace) - _, path = tempfile.mkstemp(dir=os.path.join(private_data_dir, 'env')) - with open(path, 'w') as f: - f.write(data) - os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) - container_path = get_incontainer_path(path, private_data_dir) - - # determine if filename indicates single file or many - if file_label.find('.') == -1: - tower_namespace.filename = container_path - else: - if not hasattr(tower_namespace, 'filename'): - tower_namespace.filename = TowerNamespace() - file_label = file_label.split('.')[1] - setattr(tower_namespace.filename, file_label, container_path) - - injector_field = self._meta.get_field('injectors') - for env_var, tmpl in self.injectors.get('env', {}).items(): - try: - injector_field.validate_env_var_allowed(env_var) - except ValidationError as e: - logger.error('Ignoring prohibited env var {}, reason: {}'.format(env_var, e)) - continue - env[env_var] = sandbox_env.from_string(tmpl).render(**namespace) - safe_env[env_var] = sandbox_env.from_string(tmpl).render(**safe_namespace) - - if 'INVENTORY_UPDATE_ID' not in env: - # awx-manage inventory_update does not support extra_vars via -e - def build_extra_vars(node): - if isinstance(node, dict): - return {build_extra_vars(k): build_extra_vars(v) for k, v in node.items()} - elif isinstance(node, list): - return [build_extra_vars(x) for x in node] - else: - return sandbox_env.from_string(node).render(**namespace) - - def build_extra_vars_file(vars, private_dir): - handle, path = tempfile.mkstemp(dir=os.path.join(private_dir, 'env')) - f = os.fdopen(handle, 'w') - f.write(safe_dump(vars)) - f.close() - os.chmod(path, stat.S_IRUSR) - return path - - extra_vars = build_extra_vars(self.injectors.get('extra_vars', {})) - if extra_vars: - path = build_extra_vars_file(extra_vars, private_data_dir) - container_path = get_incontainer_path(path, private_data_dir) - args.extend(['-e', '@%s' % container_path]) + inject_credential(self, credential, env, safe_env, args, private_data_dir) class CredentialTypeHelper: @@ -690,7 +566,7 @@ def get_creation_params(cls, cred_type): @classmethod def create(cls, cred_type): res = CredentialType(**CredentialTypeHelper.get_creation_params(cred_type)) - res.custom_injectors = cred_type.custom_injectors + res.custom_injectors = getattr(cred_type, "custom_injectors", None) return res From e880b7492a71adba797345e9e85472df92908936 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 19 Dec 2024 16:29:52 -0500 Subject: [PATCH 08/18] fix cloudforms test and move credential loading --- awx/main/apps.py | 3 ++ awx/main/models/credential.py | 46 ++++++++++--------- awx/main/tests/conftest.py | 9 ++++ .../test_inventory_source_migration.py | 16 +++++-- 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/awx/main/apps.py b/awx/main/apps.py index 3d9896374743..c1c90f83c7fb 100644 --- a/awx/main/apps.py +++ b/awx/main/apps.py @@ -60,6 +60,9 @@ def _load_credential_types_feature(self): @bypass_in_test def load_credential_types_feature(self): + from awx.main.models.credential import load_credentials + + load_credentials() return self._load_credential_types_feature() def load_inventory_plugins(self): diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index c8cbb96db967..81a598310292 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -638,26 +638,30 @@ def get_absolute_url(self, request=None): return reverse(view_name, kwargs={'pk': self.pk}, request=request) -awx_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials')} -supported_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials.supported')} -plugin_entry_points = awx_entry_points if detect_server_product_name() == 'AWX' else {**awx_entry_points, **supported_entry_points} - -for ns, ep in plugin_entry_points.items(): - cred_plugin = ep.load() - if not hasattr(cred_plugin, 'inputs'): - setattr(cred_plugin, 'inputs', {}) - if not hasattr(cred_plugin, 'injectors'): - setattr(cred_plugin, 'injectors', {}) - if ns in ManagedCredentialType.registry: - raise ValueError( - 'a ManagedCredentialType with namespace={} is already defined in {}'.format(ns, inspect.getsourcefile(ManagedCredentialType.registry[ns].__class__)) - ) - ManagedCredentialType.registry[ns] = cred_plugin +def load_credentials(): + + awx_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials')} + supported_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials.supported')} + plugin_entry_points = awx_entry_points if detect_server_product_name() == 'AWX' else {**awx_entry_points, **supported_entry_points} + + for ns, ep in plugin_entry_points.items(): + cred_plugin = ep.load() + if not hasattr(cred_plugin, 'inputs'): + setattr(cred_plugin, 'inputs', {}) + if not hasattr(cred_plugin, 'injectors'): + setattr(cred_plugin, 'injectors', {}) + if ns in ManagedCredentialType.registry: + raise ValueError( + 'a ManagedCredentialType with namespace={} is already defined in {}'.format( + ns, inspect.getsourcefile(ManagedCredentialType.registry[ns].__class__) + ) + ) + ManagedCredentialType.registry[ns] = cred_plugin -credential_plugins = {ep.name: ep for ep in entry_points(group='awx_plugins.credentials')} -if detect_server_product_name() == 'AWX': - credential_plugins = {} + credential_plugins = {ep.name: ep for ep in entry_points(group='awx_plugins.credentials')} + if detect_server_product_name() == 'AWX': + credential_plugins = {} -for ns, ep in credential_plugins.items(): - plugin = ep.load() - CredentialType.load_plugin(ns, plugin) + for ns, ep in credential_plugins.items(): + plugin = ep.load() + CredentialType.load_plugin(ns, plugin) diff --git a/awx/main/tests/conftest.py b/awx/main/tests/conftest.py index e1a1c05e00e8..fbfb6489f009 100644 --- a/awx/main/tests/conftest.py +++ b/awx/main/tests/conftest.py @@ -229,3 +229,12 @@ def me_inst(): me_mock = mock.MagicMock(return_value=inst) with mock.patch.object(Instance.objects, 'me', me_mock): yield inst + + +@pytest.fixture(scope="session", autouse=True) +def load_all_credentials(): + with mock.patch('awx.main.models.credential.detect_server_product_name', return_value='NOT_AWX'): + from awx.main.models.credential import load_credentials + + load_credentials() + yield diff --git a/awx/main/tests/functional/migrations/test_inventory_source_migration.py b/awx/main/tests/functional/migrations/test_inventory_source_migration.py index fe0b537426d2..6d17e22936cf 100644 --- a/awx/main/tests/functional/migrations/test_inventory_source_migration.py +++ b/awx/main/tests/functional/migrations/test_inventory_source_migration.py @@ -37,16 +37,24 @@ def cleanup_cloudforms(): assert 'cloudforms' not in CredentialType.defaults -@pytest.mark.django_db -def test_cloudforms_inventory_removal(request, inventory): - request.addfinalizer(cleanup_cloudforms) - ManagedCredentialType( +@pytest.fixture +def cloudforms_mct(): + ManagedCredentialType.registry['cloudforms'] = ManagedCredentialType( name='Red Hat CloudForms', namespace='cloudforms', kind='cloud', managed=True, inputs={}, + injectors={}, ) + yield + ManagedCredentialType.registry.pop('cloudforms', None) + + +@pytest.mark.django_db +def test_cloudforms_inventory_removal(request, inventory, cloudforms_mct): + request.addfinalizer(cleanup_cloudforms) + CredentialType.defaults['cloudforms']().save() cloudforms = CredentialType.objects.get(namespace='cloudforms') Credential.objects.create( From 84aa557228d7c61e88027b3d99ee19e1c5a8e9e5 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Tue, 7 Jan 2025 14:51:08 -0500 Subject: [PATCH 09/18] fix linting issues around undefined var --- awx/main/models/credential.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 81a598310292..17113427fccb 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -462,8 +462,7 @@ def askable_fields(self): def plugin(self): if self.kind != 'external': raise AttributeError('plugin') - [plugin] = [plugin for ns, plugin in credential_plugins.items() if ns == self.namespace] - return plugin + return ManagedCredentialType.registry.get(self.namespace, None) def default_for_field(self, field_id): for field in self.inputs.get('fields', []): From 0e9be8f3aff51c1e05697c1139e6d6b00965c6b2 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Tue, 7 Jan 2025 15:50:29 -0500 Subject: [PATCH 10/18] add load credentials method to get tests passing --- awx_collection/test/awx/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awx_collection/test/awx/conftest.py b/awx_collection/test/awx/conftest.py index 42500342ac9c..1ecdbb89f976 100644 --- a/awx_collection/test/awx/conftest.py +++ b/awx_collection/test/awx/conftest.py @@ -18,6 +18,7 @@ from ansible.module_utils.six import raise_from from ansible_base.rbac.models import RoleDefinition, DABPermission +from awx.main.tests.conftest import load_all_credentials from awx.main.tests.functional.conftest import _request from awx.main.tests.functional.conftest import credentialtype_scm, credentialtype_ssh # noqa: F401; pylint: disable=unused-import from awx.main.models import ( From 104877f486863265b1fd60efc68a95ea95f06ff5 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Wed, 8 Jan 2025 10:36:48 -0500 Subject: [PATCH 11/18] fix lint, please squash --- awx_collection/test/awx/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/awx_collection/test/awx/conftest.py b/awx_collection/test/awx/conftest.py index 1ecdbb89f976..461bc74f88c6 100644 --- a/awx_collection/test/awx/conftest.py +++ b/awx_collection/test/awx/conftest.py @@ -364,3 +364,6 @@ def rrule(): @pytest.fixture def schedule(job_template, rrule): return Schedule.objects.create(unified_job_template=job_template, name='test-sched', rrule=rrule) + +# has autouse on for the fixture but calling to satisfy linting +oad_all_credentials() From fffe5a111489ec7493888a844942ed085c141253 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Wed, 8 Jan 2025 10:39:11 -0500 Subject: [PATCH 12/18] add skip comment and revert function call --- awx_collection/test/awx/conftest.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/awx_collection/test/awx/conftest.py b/awx_collection/test/awx/conftest.py index 461bc74f88c6..dfb9daa89ac2 100644 --- a/awx_collection/test/awx/conftest.py +++ b/awx_collection/test/awx/conftest.py @@ -18,7 +18,7 @@ from ansible.module_utils.six import raise_from from ansible_base.rbac.models import RoleDefinition, DABPermission -from awx.main.tests.conftest import load_all_credentials +from awx.main.tests.conftest import load_all_credentials # noqa: F401 from awx.main.tests.functional.conftest import _request from awx.main.tests.functional.conftest import credentialtype_scm, credentialtype_ssh # noqa: F401; pylint: disable=unused-import from awx.main.models import ( @@ -364,6 +364,3 @@ def rrule(): @pytest.fixture def schedule(job_template, rrule): return Schedule.objects.create(unified_job_template=job_template, name='test-sched', rrule=rrule) - -# has autouse on for the fixture but calling to satisfy linting -oad_all_credentials() From 878c88721f6aa9127e74384b8a78378acbaa2f8a Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Wed, 8 Jan 2025 14:00:00 -0500 Subject: [PATCH 13/18] add a line to requirements file to rebuild --- requirements/requirements_git.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt index 9818bb7f529b..f7af9c0036b5 100644 --- a/requirements/requirements_git.txt +++ b/requirements/requirements_git.txt @@ -4,3 +4,5 @@ git+https://github.com/ansible/ansible-runner.git@devel#egg=ansible-runner django-ansible-base @ git+https://github.com/ansible/django-ansible-base@devel#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac] awx-plugins-core @ git+https://git@github.com/ansible/awx-plugins.git@devel#egg=awx-plugins-core awx_plugins.interfaces @ git+https://github.com/ansible/awx_plugins.interfaces.git + +# remove line after CI is green, here for cache busting purposes From 71dc780802b649de2cfdfa9ea8b63fdc6a435533 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Wed, 8 Jan 2025 15:06:00 -0500 Subject: [PATCH 14/18] fix linter please, squash me --- awx_collection/test/awx/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx_collection/test/awx/conftest.py b/awx_collection/test/awx/conftest.py index dfb9daa89ac2..6de5ed9c25ba 100644 --- a/awx_collection/test/awx/conftest.py +++ b/awx_collection/test/awx/conftest.py @@ -18,7 +18,7 @@ from ansible.module_utils.six import raise_from from ansible_base.rbac.models import RoleDefinition, DABPermission -from awx.main.tests.conftest import load_all_credentials # noqa: F401 +from awx.main.tests.conftest import load_all_credentials # noqa: F401; pylint: disable=unused-import from awx.main.tests.functional.conftest import _request from awx.main.tests.functional.conftest import credentialtype_scm, credentialtype_ssh # noqa: F401; pylint: disable=unused-import from awx.main.models import ( From ec784bc4718c2657dcc6412d8d48c07063fcf9be Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Thu, 9 Jan 2025 10:03:41 -0500 Subject: [PATCH 15/18] inject whitespace to force rebuild and load creds in live test --- awx/main/tests/live/tests/conftest.py | 1 + requirements/requirements_git.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/main/tests/live/tests/conftest.py b/awx/main/tests/live/tests/conftest.py index 0fae03918971..f7c197199f8b 100644 --- a/awx/main/tests/live/tests/conftest.py +++ b/awx/main/tests/live/tests/conftest.py @@ -5,6 +5,7 @@ # These tests are invoked from the awx/main/tests/live/ subfolder # so any fixtures from higher-up conftest files must be explicitly included from awx.main.tests.functional.conftest import * # noqa +from awx.main.tests.conftest import load_all_credentials # noqa: F401; pylint: disable=unused-import from awx.main.models import Organization diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt index f7af9c0036b5..ae68c05488a2 100644 --- a/requirements/requirements_git.txt +++ b/requirements/requirements_git.txt @@ -2,7 +2,7 @@ git+https://github.com/ansible/system-certifi.git@devel#egg=certifi # Remove pbr from requirements.in when moving ansible-runner to requirements.in git+https://github.com/ansible/ansible-runner.git@devel#egg=ansible-runner django-ansible-base @ git+https://github.com/ansible/django-ansible-base@devel#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac] -awx-plugins-core @ git+https://git@github.com/ansible/awx-plugins.git@devel#egg=awx-plugins-core +awx-plugins-core @ git+https://github.com/ansible/awx-plugins.git@devel#egg=awx-plugins-core awx_plugins.interfaces @ git+https://github.com/ansible/awx_plugins.interfaces.git # remove line after CI is green, here for cache busting purposes From b3a2b16460ebac1eb997c18f183ac549dc6f4291 Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Fri, 10 Jan 2025 14:33:27 -0500 Subject: [PATCH 16/18] update test to run cred tests when cred is present --- .../targets/credential/tasks/main.yml | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/awx_collection/tests/integration/targets/credential/tasks/main.yml b/awx_collection/tests/integration/targets/credential/tasks/main.yml index 34dd058d97d1..019b5abab09b 100644 --- a/awx_collection/tests/integration/targets/credential/tasks/main.yml +++ b/awx_collection/tests/integration/targets/credential/tasks/main.yml @@ -24,6 +24,28 @@ insights_cred_name1: "AWX-Collection-tests-credential-insights-cred1-{{ test_id }}" tower_cred_name1: "AWX-Collection-tests-credential-tower-cred1-{{ test_id }}" +- name: Get current Credential Types available + ansible.builtin.uri: + url: "https://localhost:8043/api/v2/credential_types/" + force_basic_auth: true + user: admin + password: admin + method: GET + return_content: yes + validate_certs: false + register: credentials + +- name: Register Credentials found + set_fact: + aws_found: "{{ 'Amazon Web Services' in credentials.json.results | map(attribute='name') | list }}" + vmware_found: "{{ 'VMware vCenter' in credentials.json.results | map(attribute='name') | list }}" + azure_found: "{{ 'Microsoft Azure Resource Manager' in credentials.json.results | map(attribute='name') | list }}" + gce_found: "{{ 'Google Compute Engine' in credentials.json.results | map(attribute='name') | list }}" + insights_found: "{{ 'Red Hat Insights' in credentials.json.results | map(attribute='name') | list }}" + satellite_found: "{{ 'Red Hat Satellite 6' in credentials.json.results | map(attribute='name') | list }}" + openstack_found: "{{ 'OpenStack' in credentials.json.results | map(attribute='name') | list }}" + rhv_found: "{{ 'Red Hat Virtualization' in credentials.json.results | map(attribute='name') | list }}" + - name: create a tempdir for an SSH key local_action: shell mktemp -d register: tempdir @@ -451,7 +473,7 @@ - assert: that: - "result is changed" - + - name: Create a valid AWS credential credential: name: "{{ aws_cred_name1 }}" @@ -463,10 +485,12 @@ password: secret security_token: aws-token register: result + when: aws_found - assert: that: - "result is changed" + when: aws_found - name: Delete an AWS credential credential: @@ -475,10 +499,12 @@ state: absent credential_type: Amazon Web Services register: result + when: aws_found - assert: that: - "result is changed" + when: aws_found - name: Create a valid VMWare credential credential: @@ -491,10 +517,12 @@ username: joe password: secret register: result + when: vmware_found - assert: that: - "result is changed" + when: vmware_found - name: Delete an VMWare credential credential: @@ -503,10 +531,12 @@ state: absent credential_type: VMware vCenter register: result + when: vmware_found - assert: that: - "result is changed" + when: vmware_found - name: Create a valid Satellite6 credential credential: @@ -519,10 +549,12 @@ username: joe password: secret register: result + when: satellite_found - assert: that: - "result is changed" + when: satellite_found - name: Delete a Satellite6 credential credential: @@ -531,10 +563,12 @@ state: absent credential_type: Red Hat Satellite 6 register: result + when: satellite_found - assert: that: - "result is changed" + when: satellite_found - name: Create a valid GCE credential credential: @@ -547,10 +581,12 @@ project: ABC123 ssh_key_data: "{{ ssh_key_data }}" register: result + when: gce_found - assert: that: - "result is changed" + when: gce_found - name: Delete a GCE credential credential: @@ -559,10 +595,12 @@ state: absent credential_type: Google Compute Engine register: result + when: gce_found - assert: that: - "result is changed" + when: gce_found - name: Create a valid AzureRM credential credential: @@ -575,10 +613,12 @@ password: secret subscription: some-subscription register: result + when: azure_found - assert: that: - "result is changed" + when: azure_found - name: Create a valid AzureRM credential with a tenant credential: @@ -592,10 +632,12 @@ tenant: some-tenant subscription: some-subscription register: result + when: azure_found - assert: that: - "result is changed" + when: azure_found - name: Delete an AzureRM credential credential: @@ -604,10 +646,12 @@ state: absent credential_type: Microsoft Azure Resource Manager register: result + when: azure_found - assert: that: - "result is changed" + when: azure_found - name: Create a valid OpenStack credential credential: @@ -622,10 +666,12 @@ project: tenant123 domain: some-domain register: result + when: openstack_found - assert: that: - "result is changed" + when: openstack_found - name: Delete a OpenStack credential credential: @@ -634,10 +680,12 @@ state: absent credential_type: OpenStack register: result + when: openstack_found - assert: that: - "result is changed" + when: openstack_found - name: Create a valid RHV credential credential: @@ -650,10 +698,12 @@ username: joe password: secret register: result + when: rhv_found - assert: that: - "result is changed" + when: rhv_found - name: Delete an RHV credential credential: @@ -662,10 +712,12 @@ state: absent credential_type: Red Hat Virtualization register: result + when: rhv_found - assert: that: - "result is changed" + when: rhv_found - name: Create a valid Insights credential credential: @@ -677,10 +729,12 @@ username: joe password: secret register: result + when: insights_found - assert: that: - "result is changed" + when: insights_found - name: Delete an Insights credential credential: @@ -689,10 +743,12 @@ state: absent credential_type: Insights register: result + when: insights_found - assert: that: - "result is changed" + when: insights_found - name: Create a valid Tower-to-Tower credential credential: From ba44a0867184d1be0bae6af4ad823cd3e848a7af Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Fri, 10 Jan 2025 14:39:52 -0500 Subject: [PATCH 17/18] hide username, pass and verify certs. squash me --- .../tests/integration/targets/credential/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/awx_collection/tests/integration/targets/credential/tasks/main.yml b/awx_collection/tests/integration/targets/credential/tasks/main.yml index 019b5abab09b..2d9757ac2fa2 100644 --- a/awx_collection/tests/integration/targets/credential/tasks/main.yml +++ b/awx_collection/tests/integration/targets/credential/tasks/main.yml @@ -28,11 +28,11 @@ ansible.builtin.uri: url: "https://localhost:8043/api/v2/credential_types/" force_basic_auth: true - user: admin - password: admin + user: user + password: password method: GET return_content: yes - validate_certs: false + validate_certs: true register: credentials - name: Register Credentials found From 5ab445ee90385b03f80e8ccf94b35236644011ff Mon Sep 17 00:00:00 2001 From: thedoubl3j Date: Mon, 13 Jan 2025 09:45:28 -0500 Subject: [PATCH 18/18] update tests with correct u/p, remove requirement --- .../tests/integration/targets/credential/tasks/main.yml | 6 +++--- requirements/requirements_git.txt | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/awx_collection/tests/integration/targets/credential/tasks/main.yml b/awx_collection/tests/integration/targets/credential/tasks/main.yml index 2d9757ac2fa2..aa5eee661580 100644 --- a/awx_collection/tests/integration/targets/credential/tasks/main.yml +++ b/awx_collection/tests/integration/targets/credential/tasks/main.yml @@ -28,11 +28,11 @@ ansible.builtin.uri: url: "https://localhost:8043/api/v2/credential_types/" force_basic_auth: true - user: user + user: admin password: password method: GET return_content: yes - validate_certs: true + validate_certs: false register: credentials - name: Register Credentials found @@ -473,7 +473,7 @@ - assert: that: - "result is changed" - + - name: Create a valid AWS credential credential: name: "{{ aws_cred_name1 }}" diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt index ae68c05488a2..07e0b8878c02 100644 --- a/requirements/requirements_git.txt +++ b/requirements/requirements_git.txt @@ -4,5 +4,3 @@ git+https://github.com/ansible/ansible-runner.git@devel#egg=ansible-runner django-ansible-base @ git+https://github.com/ansible/django-ansible-base@devel#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac] awx-plugins-core @ git+https://github.com/ansible/awx-plugins.git@devel#egg=awx-plugins-core awx_plugins.interfaces @ git+https://github.com/ansible/awx_plugins.interfaces.git - -# remove line after CI is green, here for cache busting purposes