Merge pull request #22 from ctrliq/devel

Sync to Upstream 24.0.0

.github/workflows/devel_images.yml.rej

@@ -0,0 +1,10 @@
+--- .github/workflows/devel_images.yml
++++ .github/workflows/devel_images.yml
+@@ -3,6 +3,7 @@ name: Build/Push Development Images
+ env:
+   LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
+ on:
++  workflow_dispatch:
+   push:
+     branches:
+       - devel

Makefile

@@ -216,16 +216,14 @@ collectstatic:
 	fi; \
 	$(PYTHON) manage.py collectstatic --clear --noinput > /dev/null 2>&1
 
-DEV_RELOAD_COMMAND ?= supervisorctl restart tower-processes:*
-
 uwsgi: collectstatic
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
 	uwsgi /etc/tower/uwsgi.ini
 
 awx-autoreload:
-	@/awx_devel/tools/docker-compose/awx-autoreload /awx_devel/awx "$(DEV_RELOAD_COMMAND)"
+	@/awx_devel/tools/docker-compose/awx-autoreload /awx_devel/awx
 
 daphne:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -305,7 +303,7 @@ swagger: reports
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	(set -o pipefail && py.test $(PYTEST_ARGS) awx/conf/tests/functional awx/main/tests/functional/api awx/main/tests/docs --release=$(VERSION_TARGET) | tee reports/$@.report)
+	(set -o pipefail && py.test $(PYTEST_ARGS) awx/conf/tests/functional awx/main/tests/functional/api awx/main/tests/docs | tee reports/$@.report)
 
 check: black
 
@@ -631,9 +629,6 @@ clean-elk:
 	docker rm tools_elasticsearch_1
 	docker rm tools_kibana_1
 
-psql-container:
-	docker run -it --net tools_default --rm postgres:12 sh -c 'exec psql -h "postgres" -p "5432" -U postgres'
-
 VERSION:
 	@echo "awx: $(VERSION)"
 

awx/__init__.py

@@ -154,10 +154,12 @@ def manage():
     from django.conf import settings
     from django.core.management import execute_from_command_line
 
-    # enforce the postgres version is equal to 12. if not, then terminate program with exit code of 1
+    # enforce the postgres version is a minimum of 12 (we need this for partitioning); if not, then terminate program with exit code of 1
+    # In the future if we require a feature of a version of postgres > 12 this should be updated to reflect that.
+    # The return of connection.pg_version is something like 12013
     if not os.getenv('SKIP_PG_VERSION_CHECK', False) and not MODE == 'development':
         if (connection.pg_version // 10000) < 12:
-            sys.stderr.write("Postgres version 12 is required\n")
+            sys.stderr.write("At a minimum, postgres version 12 is required\n")
             sys.exit(1)
 
     if len(sys.argv) >= 2 and sys.argv[1] in ('version', '--version'):  # pragma: no cover

awx/conf/settings.py

@@ -1,6 +1,7 @@
 # Python
 import contextlib
 import logging
+import psycopg
 import threading
 import time
 import os
@@ -13,7 +14,7 @@
 from django.core.cache import cache as django_cache
 from django.core.exceptions import ImproperlyConfigured, SynchronousOnlyOperation
 from django.db import transaction, connection
-from django.db.utils import Error as DBError, ProgrammingError
+from django.db.utils import DatabaseError, ProgrammingError
 from django.utils.functional import cached_property
 
 # Django REST Framework
@@ -80,18 +81,26 @@ def _ctit_db_wrapper(trans_safe=False):
                     logger.debug('Obtaining database settings in spite of broken transaction.')
                     transaction.set_rollback(False)
         yield
-    except DBError as exc:
+    except ProgrammingError as e:
+        # Exception raised for programming errors
+        # Examples may be table not found or already exists,
+        # this generally means we can't fetch Tower configuration
+        # because the database hasn't actually finished migrating yet;
+        # this is usually a sign that a service in a container (such as ws_broadcast)
+        # has come up *before* the database has finished migrating, and
+        # especially that the conf.settings table doesn't exist yet
+        # syntax error in the SQL statement, wrong number of parameters specified, etc.
         if trans_safe:
-            level = logger.warning
-            if isinstance(exc, ProgrammingError):
-                if 'relation' in str(exc) and 'does not exist' in str(exc):
-                    # this generally means we can't fetch Tower configuration
-                    # because the database hasn't actually finished migrating yet;
-                    # this is usually a sign that a service in a container (such as ws_broadcast)
-                    # has come up *before* the database has finished migrating, and
-                    # especially that the conf.settings table doesn't exist yet
-                    level = logger.debug
-            level(f'Database settings are not available, using defaults. error: {str(exc)}')
+            logger.debug(f'Database settings are not available, using defaults. error: {str(e)}')
+        else:
+            logger.exception('Error modifying something related to database settings.')
+    except DatabaseError as e:
+        if trans_safe:
+            cause = e.__cause__
+            if cause and hasattr(cause, 'sqlstate'):
+                sqlstate = cause.sqlstate
+                sqlstate_str = psycopg.errors.lookup(sqlstate)
+                logger.error('SQL Error state: {} - {}'.format(sqlstate, sqlstate_str))
         else:
             logger.exception('Error modifying something related to database settings.')
     finally:

awx/main/analytics/collectors.py

@@ -419,7 +419,7 @@ def query(event_data):
                           resolved_action,
                           resolved_role,
                           -- '-' operator listed here:
-                          -- https://www.postgresql.org/docs/12/functions-json.html
+                          -- https://www.postgresql.org/docs/15/functions-json.html
                           -- note that operator is only supported by jsonb objects
                           -- https://www.postgresql.org/docs/current/datatype-json.html
                           (CASE WHEN event = 'playbook_on_stats' THEN {event_data} - 'artifact_data' END) as playbook_on_stats,

awx/main/constants.py

@@ -14,7 +14,7 @@
     'STANDARD_INVENTORY_UPDATE_ENV',
 ]
 
-CLOUD_PROVIDERS = ('azure_rm', 'ec2', 'gce', 'vmware', 'openstack', 'rhv', 'satellite6', 'controller', 'insights')
+CLOUD_PROVIDERS = ('azure_rm', 'ec2', 'gce', 'vmware', 'openstack', 'rhv', 'satellite6', 'controller', 'insights', 'terraform')
 PRIVILEGE_ESCALATION_METHODS = [
     ('sudo', _('Sudo')),
     ('su', _('Su')),

awx/main/migrations/0190_alter_inventorysource_source_and_more.py

@@ -0,0 +1,53 @@
+# Generated by Django 4.2.6 on 2024-02-15 20:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0189_inbound_hop_nodes'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='inventorysource',
+            name='source',
+            field=models.CharField(
+                choices=[
+                    ('file', 'File, Directory or Script'),
+                    ('constructed', 'Template additional groups and hostvars at runtime'),
+                    ('scm', 'Sourced from a Project'),
+                    ('ec2', 'Amazon EC2'),
+                    ('gce', 'Google Compute Engine'),
+                    ('azure_rm', 'Microsoft Azure Resource Manager'),
+                    ('vmware', 'VMware vCenter'),
+                    ('openstack', 'OpenStack'),
+                    ('controller', 'CIQ Ascender Automation Platform'),
+                    ('terraform', 'Terraform State'),
+                ],
+                default=None,
+                max_length=32,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='inventoryupdate',
+            name='source',
+            field=models.CharField(
+                choices=[
+                    ('file', 'File, Directory or Script'),
+                    ('constructed', 'Template additional groups and hostvars at runtime'),
+                    ('scm', 'Sourced from a Project'),
+                    ('ec2', 'Amazon EC2'),
+                    ('gce', 'Google Compute Engine'),
+                    ('azure_rm', 'Microsoft Azure Resource Manager'),
+                    ('vmware', 'VMware vCenter'),
+                    ('openstack', 'OpenStack'),
+                    ('controller', 'CIQ Ascender Automation Platform'),
+                    ('terraform', 'Terraform State'),
+                ],
+                default=None,
+                max_length=32,
+            ),
+        ),
+    ]

awx/main/models/inventory.py

@@ -925,6 +925,7 @@ class InventorySourceOptions(BaseModel):
         ('vmware', _('VMware vCenter')),
         # ('satellite6', _('Red Hat Satellite 6')),
         ('openstack', _('OpenStack')),
+        ('terraform', _('Terraform State')),
         # ('rhv', _('Red Hat Virtualization')),
         # ('controller', _('Red Hat Ansible Automation Platform')),
         # ('insights', _('Red Hat Insights')),
@@ -1633,6 +1634,20 @@ def get_plugin_env(self, inventory_update, private_data_dir, private_data_files)
         return ret
 
 
+class terraform(PluginFileInjector):
+    plugin_name = 'terraform_state'
+    base_injector = 'managed'
+    namespace = 'cloud'
+    collection = 'terraform'
+    use_fqcn = True
+
+    def inventory_as_dict(self, inventory_update, private_data_dir):
+        env = super(terraform, self).get_plugin_env(inventory_update, private_data_dir, None)
+        ret = super().inventory_as_dict(inventory_update, private_data_dir)
+        ret['backend_config_files'] = env["TF_BACKEND_CONFIG_FILE"]
+        return ret
+
+
 class controller(PluginFileInjector):
     plugin_name = 'tower'  # TODO: relying on routing for now, update after EEs pick up revised collection
     base_injector = 'template'

awx/main/tasks/system.py

@@ -6,6 +6,7 @@
 import json
 import logging
 import os
+import psycopg
 from io import StringIO
 from contextlib import redirect_stdout
 import shutil
@@ -416,7 +417,7 @@ def handle_removed_image(remove_images=None):
 
 @task(queue=get_task_queuename)
 def cleanup_images_and_files():
-    _cleanup_images_and_files()
+    _cleanup_images_and_files(image_prune=True)
 
 
 @task(queue=get_task_queuename)
@@ -630,10 +631,18 @@ def cluster_node_heartbeat(dispatch_time=None, worker_tasks=None):
                 logger.error("Host {} last checked in at {}, marked as lost.".format(other_inst.hostname, other_inst.last_seen))
 
         except DatabaseError as e:
-            if 'did not affect any rows' in str(e):
-                logger.debug('Another instance has marked {} as lost'.format(other_inst.hostname))
+            cause = e.__cause__
+            if cause and hasattr(cause, 'sqlstate'):
+                sqlstate = cause.sqlstate
+                sqlstate_str = psycopg.errors.lookup(sqlstate)
+                logger.debug('SQL Error state: {} - {}'.format(sqlstate, sqlstate_str))
+
+                if sqlstate == psycopg.errors.NoData:
+                    logger.debug('Another instance has marked {} as lost'.format(other_inst.hostname))
+                else:
+                    logger.exception("Error marking {} as lost.".format(other_inst.hostname))
             else:
-                logger.exception('Error marking {} as lost'.format(other_inst.hostname))
+                logger.exception('No SQL state available.  Error marking {} as lost'.format(other_inst.hostname))
 
     # Run local reaper
     if worker_tasks is not None:
@@ -788,10 +797,19 @@ def update_inventory_computed_fields(inventory_id):
     try:
         i.update_computed_fields()
     except DatabaseError as e:
-        if 'did not affect any rows' in str(e):
-            logger.debug('Exiting duplicate update_inventory_computed_fields task.')
-            return
-        raise
+        # https://github.com/django/django/blob/eff21d8e7a1cb297aedf1c702668b590a1b618f3/django/db/models/base.py#L1105
+        # django raises DatabaseError("Forced update did not affect any rows.")
+
+        # if sqlstate is set then there was a database error and otherwise will re-raise that error
+        cause = e.__cause__
+        if cause and hasattr(cause, 'sqlstate'):
+            sqlstate = cause.sqlstate
+            sqlstate_str = psycopg.errors.lookup(sqlstate)
+            logger.error('SQL Error state: {} - {}'.format(sqlstate, sqlstate_str))
+            raise
+
+        # otherwise
+        logger.debug('Exiting duplicate update_inventory_computed_fields task.')
 
 
 def update_smart_memberships_for_inventory(smart_inventory):

awx/main/tests/data/inventory/plugins/terraform/env.json

@@ -0,0 +1,3 @@
+{
+    "TF_BACKEND_CONFIG_FILE": "{{ file_reference }}"
+}

awx/main/tests/docs/conftest.py

@@ -1,13 +1,8 @@
 from awx.main.tests.functional.conftest import *  # noqa
+import os
+import pytest
 
 
-def pytest_addoption(parser):
-    parser.addoption("--release", action="store", help="a release version number, e.g., 3.3.0")
-
-
-def pytest_generate_tests(metafunc):
-    # This is called for every test. Only get/set command line arguments
-    # if the argument is specified in the list of test "fixturenames".
-    option_value = metafunc.config.option.release
-    if 'release' in metafunc.fixturenames and option_value is not None:
-        metafunc.parametrize("release", [option_value])
+@pytest.fixture()
+def release():
+    return os.environ.get('VERSION_TARGET', '')

awx/main/tests/functional/conftest.py

@@ -3,15 +3,19 @@
 from unittest import mock
 import urllib.parse
 from unittest.mock import PropertyMock
+import importlib
 
 # Django
 from django.urls import resolve
 from django.http import Http404
+from django.apps import apps
 from django.core.handlers.exception import response_for_exception
 from django.contrib.auth.models import User
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db.backends.sqlite3.base import SQLiteCursorWrapper
 
+from django.db.models.signals import post_migrate
+
 # AWX
 from awx.main.models.projects import Project
 from awx.main.models.ha import Instance
@@ -41,10 +45,19 @@
 from awx.main.models.ad_hoc_commands import AdHocCommand
 from awx.main.models.oauth import OAuth2Application as Application
 from awx.main.models.execution_environments import ExecutionEnvironment
+from awx.main.utils import is_testing
 
 __SWAGGER_REQUESTS__ = {}
 
 
+# HACK: the dab_resource_registry app required ServiceID in migrations which checks do not run
+dab_rr_initial = importlib.import_module('ansible_base.resource_registry.migrations.0001_initial')
+
+
+if is_testing():
+    post_migrate.connect(lambda **kwargs: dab_rr_initial.create_service_id(apps, None))
+
+
 @pytest.fixture(scope="session")
 def swagger_autogen(requests=__SWAGGER_REQUESTS__):
     return requests

awx/main/tests/functional/models/test_inventory.py

@@ -195,6 +195,7 @@ def test_plugin_filenames(self, source, filename):
             ('satellite6', 'theforeman.foreman.foreman'),
             # ('insights', 'redhatinsights.insights.insights'),
             ('controller', 'awx.awx.tower'),
+            ('terraform', 'cloud.terraform.terraform_state'),
         ],
     )
     def test_plugin_proper_names(self, source, proper_name):

awx/main/tests/functional/test_inventory_source_injectors.py

@@ -107,6 +107,7 @@ def read_content(private_data_dir, raw_env, inventory_update):
             for filename in os.listdir(os.path.join(private_data_dir, subdir)):
                 filename_list.append(os.path.join(subdir, filename))
     filename_list = sorted(filename_list, key=lambda fn: inverse_env.get(os.path.join(private_data_dir, fn), [fn])[0])
+    inventory_content = ""
     for filename in filename_list:
         if filename in ('args', 'project'):
             continue  # Ansible runner
@@ -130,6 +131,7 @@ def read_content(private_data_dir, raw_env, inventory_update):
                 dir_contents[abs_file_path] = f.read()
             # Declare a reference to inventory plugin file if it exists
             if abs_file_path.endswith('.yml') and 'plugin: ' in dir_contents[abs_file_path]:
+                inventory_content = dir_contents[abs_file_path]
                 referenced_paths.add(abs_file_path)  # used as inventory file
             elif cache_file_regex.match(abs_file_path):
                 file_aliases[abs_file_path] = 'cache_file'
@@ -157,7 +159,11 @@ def read_content(private_data_dir, raw_env, inventory_update):
     content = {}
     for abs_file_path, file_content in dir_contents.items():
         # assert that all files laid down are used
-        if abs_file_path not in referenced_paths and abs_file_path not in ignore_files:
+        if (
+            abs_file_path not in referenced_paths
+            and to_container_path(abs_file_path, private_data_dir) not in inventory_content
+            and abs_file_path not in ignore_files
+        ):
             raise AssertionError(
                 "File {} is not referenced. References and files:\n{}\n{}".format(abs_file_path, json.dumps(env, indent=4), json.dumps(dir_contents, indent=4))
             )