From 35c881342539b2968f8fede02eb6cf8c470f7749 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 2 Dec 2024 10:43:25 -0500 Subject: [PATCH 1/7] Use advisory_lock from DAB --- awx/main/analytics/core.py | 4 ++- .../commands/deprovision_instance.py | 3 +- .../management/commands/inventory_import.py | 4 ++- .../management/commands/register_queue.py | 3 +- .../management/commands/unregister_queue.py | 3 +- awx/main/managers.py | 4 ++- awx/main/models/credential.py | 4 ++- awx/main/scheduler/task_manager.py | 4 ++- awx/main/tasks/receptor.py | 4 ++- awx/main/tasks/system.py | 2 +- awx/main/utils/pglock.py | 29 ------------------- requirements/requirements_git.txt | 2 +- 12 files changed, 26 insertions(+), 40 deletions(-) delete mode 100644 awx/main/utils/pglock.py diff --git a/awx/main/analytics/core.py b/awx/main/analytics/core.py index 1195b262ba20..9efe27337c29 100644 --- a/awx/main/analytics/core.py +++ b/awx/main/analytics/core.py @@ -16,10 +16,12 @@ import requests from awx.conf.license import get_license + +from ansible_base.lib.utils.db import advisory_lock + from awx.main.models import Job from awx.main.access import access_registry from awx.main.utils import get_awx_http_client_headers, set_environ, datetime_hook -from awx.main.utils.pglock import advisory_lock __all__ = ['register', 'gather', 'ship'] diff --git a/awx/main/management/commands/deprovision_instance.py b/awx/main/management/commands/deprovision_instance.py index 470694d36310..ff906e2030c6 100644 --- a/awx/main/management/commands/deprovision_instance.py +++ b/awx/main/management/commands/deprovision_instance.py @@ -4,8 +4,9 @@ from django.db import transaction from django.core.management.base import BaseCommand, CommandError +from ansible_base.lib.utils.db import advisory_lock + from awx.main.models import Instance -from awx.main.utils.pglock import advisory_lock class Command(BaseCommand): diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 582af9d03bfe..29ab6ba6272e 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -21,6 +21,9 @@ # DRF error class to distinguish license exceptions from rest_framework.exceptions import PermissionDenied +# django-ansible-base +from ansible_base.lib.utils.db import advisory_lock + # AWX inventory imports from awx.main.models.inventory import Inventory, InventorySource, InventoryUpdate, Host from awx.main.utils.mem_inventory import MemInventory, dict_to_mem_data @@ -32,7 +35,6 @@ from awx.main.utils.execution_environments import get_default_execution_environment from awx.main.signals import disable_activity_stream from awx.main.constants import STANDARD_INVENTORY_UPDATE_ENV -from awx.main.utils.pglock import advisory_lock logger = logging.getLogger('awx.main.commands.inventory_import') diff --git a/awx/main/management/commands/register_queue.py b/awx/main/management/commands/register_queue.py index 36b7d17b1b9d..9b8af6e031ba 100644 --- a/awx/main/management/commands/register_queue.py +++ b/awx/main/management/commands/register_queue.py @@ -2,7 +2,8 @@ # All Rights Reserved. import sys -from awx.main.utils.pglock import advisory_lock +from ansible_base.lib.utils.db import advisory_lock + from awx.main.models import Instance, InstanceGroup from django.core.management.base import BaseCommand, CommandError diff --git a/awx/main/management/commands/unregister_queue.py b/awx/main/management/commands/unregister_queue.py index 729c30978854..f26f877d3f01 100644 --- a/awx/main/management/commands/unregister_queue.py +++ b/awx/main/management/commands/unregister_queue.py @@ -2,7 +2,8 @@ # All Rights Reserved. import sys -from awx.main.utils.pglock import advisory_lock +from ansible_base.lib.utils.db import advisory_lock + from awx.main.models import InstanceGroup from django.db import transaction diff --git a/awx/main/managers.py b/awx/main/managers.py index c501d7b0d397..c754af3b38b1 100644 --- a/awx/main/managers.py +++ b/awx/main/managers.py @@ -6,8 +6,10 @@ from django.db import models from django.conf import settings from django.db.models.functions import Lower + +from ansible_base.lib.utils.db import advisory_lock + from awx.main.utils.filters import SmartFilter -from awx.main.utils.pglock import advisory_lock from awx.main.constants import RECEPTOR_PENDING ___all__ = ['HostManager', 'InstanceManager', 'DeferJobCreatedManager', 'UUID_DEFAULT'] diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 1cdf11d135b8..09cc4bde7cb5 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -23,9 +23,11 @@ # DRF -from awx.main.utils.pglock import advisory_lock from rest_framework.serializers import ValidationError as DRFValidationError +# django-ansible-base +from ansible_base.lib.utils.db import advisory_lock + # AWX from awx.api.versioning import reverse from awx.main.fields import ( diff --git a/awx/main/scheduler/task_manager.py b/awx/main/scheduler/task_manager.py index 9ded09dd7845..835e99d4b09a 100644 --- a/awx/main/scheduler/task_manager.py +++ b/awx/main/scheduler/task_manager.py @@ -19,6 +19,9 @@ from ansible_base.lib.utils.models import get_type_for_model +# django-ansible-base +from ansible_base.lib.utils.db import advisory_lock + # AWX from awx.main.dispatch.reaper import reap_job from awx.main.models import ( @@ -34,7 +37,6 @@ WorkflowJobTemplate, ) from awx.main.scheduler.dag_workflow import WorkflowDAG -from awx.main.utils.pglock import advisory_lock from awx.main.utils import ( ScheduleTaskManager, ScheduleWorkflowManager, diff --git a/awx/main/tasks/receptor.py b/awx/main/tasks/receptor.py index 58c2e895cc46..2fbf6791ed4c 100644 --- a/awx/main/tasks/receptor.py +++ b/awx/main/tasks/receptor.py @@ -17,6 +17,9 @@ # Runner import ansible_runner +# django-ansible-base +from ansible_base.lib.utils.db import advisory_lock + # AWX from awx.main.utils.execution_environments import get_default_pod_spec from awx.main.exceptions import ReceptorNodeNotFound @@ -30,7 +33,6 @@ from awx.main.models import Instance, InstanceLink, UnifiedJob, ReceptorAddress from awx.main.dispatch import get_task_queuename from awx.main.dispatch.publish import task -from awx.main.utils.pglock import advisory_lock # Receptorctl from receptorctl.socket_interface import ReceptorControl diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index e0bb0dfc9fad..6d161d2ef8b5 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -38,6 +38,7 @@ # django-ansible-base from ansible_base.resource_registry.tasks.sync import SyncExecutor +from ansible_base.lib.utils.db import advisory_lock # AWX from awx import __version__ as awx_application_version @@ -60,7 +61,6 @@ from awx.main.utils.common import ignore_inventory_computed_fields, ignore_inventory_group_removal from awx.main.utils.reload import stop_local_services -from awx.main.utils.pglock import advisory_lock from awx.main.tasks.helpers import is_run_threshold_reached from awx.main.tasks.receptor import get_receptor_ctl, worker_info, worker_cleanup, administrative_workunit_reaper, write_receptor_config from awx.main.consumers import emit_channel_notification diff --git a/awx/main/utils/pglock.py b/awx/main/utils/pglock.py deleted file mode 100644 index d599d284b334..000000000000 --- a/awx/main/utils/pglock.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2017 Ansible by Red Hat -# All Rights Reserved. - -from contextlib import contextmanager - -from django_pglocks import advisory_lock as django_pglocks_advisory_lock -from django.db import connection - - -@contextmanager -def advisory_lock(*args, lock_session_timeout_milliseconds=0, **kwargs): - if connection.vendor == 'postgresql': - cur = None - idle_in_transaction_session_timeout = None - idle_session_timeout = None - if lock_session_timeout_milliseconds > 0: - with connection.cursor() as cur: - idle_in_transaction_session_timeout = cur.execute('SHOW idle_in_transaction_session_timeout').fetchone()[0] - idle_session_timeout = cur.execute('SHOW idle_session_timeout').fetchone()[0] - cur.execute(f"SET idle_in_transaction_session_timeout = '{lock_session_timeout_milliseconds}'") - cur.execute(f"SET idle_session_timeout = '{lock_session_timeout_milliseconds}'") - with django_pglocks_advisory_lock(*args, **kwargs) as internal_lock: - yield internal_lock - if lock_session_timeout_milliseconds > 0: - with connection.cursor() as cur: - cur.execute(f"SET idle_in_transaction_session_timeout = '{idle_in_transaction_session_timeout}'") - cur.execute(f"SET idle_session_timeout = '{idle_session_timeout}'") - else: - yield True diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt index 9818bb7f529b..f19fbbb208ca 100644 --- a/requirements/requirements_git.txt +++ b/requirements/requirements_git.txt @@ -1,6 +1,6 @@ git+https://github.com/ansible/system-certifi.git@devel#egg=certifi # Remove pbr from requirements.in when moving ansible-runner to requirements.in git+https://github.com/ansible/ansible-runner.git@devel#egg=ansible-runner -django-ansible-base @ git+https://github.com/ansible/django-ansible-base@devel#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac] +django-ansible-base @ git+https://github.com/alancoding/django-ansible-base@advisory_lock#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac] awx-plugins-core @ git+https://git@github.com/ansible/awx-plugins.git@devel#egg=awx-plugins-core awx_plugins.interfaces @ git+https://github.com/ansible/awx_plugins.interfaces.git From 1afd9ff69976e6faeff85a4dbeceb6288cfa825b Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 2 Dec 2024 10:48:13 -0500 Subject: [PATCH 2/7] Remove the django-pglocks dep --- licenses/django-pglocks.txt | 3 --- requirements/requirements.in | 1 - requirements/requirements.txt | 10 ++-------- 3 files changed, 2 insertions(+), 12 deletions(-) delete mode 100644 licenses/django-pglocks.txt diff --git a/licenses/django-pglocks.txt b/licenses/django-pglocks.txt deleted file mode 100644 index 3b74730882b5..000000000000 --- a/licenses/django-pglocks.txt +++ /dev/null @@ -1,3 +0,0 @@ -Copyright (c) 2013 Christophe Pettus - -Licensed under the MIT License. diff --git a/requirements/requirements.in b/requirements/requirements.in index 1d4de337aea4..33c6f5eec976 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -19,7 +19,6 @@ django-extensions django-guid django-oauth-toolkit<2.0.0 # Version 2.0.0 has breaking changes that will need to be worked out before upgrading django-polymorphic -django-pglocks django-solo django-split-settings djangorestframework>=3.15.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 36220721ad11..e48432d6a2db 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -23,7 +23,6 @@ asgiref==3.8.1 # channels-redis # daphne # django - # django-ansible-base # django-cors-headers asn1==2.7.1 # via -r /awx_devel/requirements/requirements.in @@ -131,7 +130,7 @@ django==4.2.16 # django-polymorphic # django-solo # djangorestframework -# django-ansible-base @ git+https://github.com/ansible/django-ansible-base@devel # git requirements installed separately +# django-ansible-base @ git+https://github.com/alancoding/django-ansible-base@advisory_lock # git requirements installed separately # via -r /awx_devel/requirements/requirements_git.txt django-cors-headers==4.6.0 # via -r /awx_devel/requirements/requirements.in @@ -147,8 +146,6 @@ django-guid==3.5.0 # via -r /awx_devel/requirements/requirements.in django-oauth-toolkit==1.7.1 # via -r /awx_devel/requirements/requirements.in -django-pglocks==1.0.4 - # via -r /awx_devel/requirements/requirements.in django-polymorphic==3.1.0 # via -r /awx_devel/requirements/requirements.in django-solo==2.4.0 @@ -371,7 +368,6 @@ pygerduty==0.38.3 pyjwt[crypto]==2.10.1 # via # adal - # django-ansible-base # msal # twilio pyopenssl==24.3.0 @@ -423,7 +419,6 @@ requests==2.32.3 # -r /awx_devel/requirements/requirements.in # adal # azure-core - # django-ansible-base # django-oauth-toolkit # kubernetes # msal @@ -456,7 +451,7 @@ setuptools-scm[toml]==8.1.0 six==1.17.0 # via # azure-core - # django-pglocks + # isodate # kubernetes # msrestazure # openshift @@ -497,7 +492,6 @@ typing-extensions==4.12.2 urllib3==2.2.3 # via # botocore - # django-ansible-base # kubernetes # requests uwsgi==2.0.28 From d87509d41297f32d0c08da63136eb008155f60eb Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 2 Dec 2024 14:29:41 -0500 Subject: [PATCH 3/7] Set back requirements after rebase --- requirements/requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index e48432d6a2db..7d61cb24100e 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -23,6 +23,7 @@ asgiref==3.8.1 # channels-redis # daphne # django + # django-ansible-base # django-cors-headers asn1==2.7.1 # via -r /awx_devel/requirements/requirements.in @@ -368,6 +369,7 @@ pygerduty==0.38.3 pyjwt[crypto]==2.10.1 # via # adal + # django-ansible-base # msal # twilio pyopenssl==24.3.0 @@ -419,6 +421,7 @@ requests==2.32.3 # -r /awx_devel/requirements/requirements.in # adal # azure-core + # django-ansible-base # django-oauth-toolkit # kubernetes # msal @@ -492,6 +495,7 @@ typing-extensions==4.12.2 urllib3==2.2.3 # via # botocore + # django-ansible-base # kubernetes # requests uwsgi==2.0.28 From d3d295dee122c32c7c78975c6072c2722d455802 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 18 Dec 2024 16:37:13 -0500 Subject: [PATCH 4/7] Set requirement back to devel --- requirements/requirements_git.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt index f19fbbb208ca..9818bb7f529b 100644 --- a/requirements/requirements_git.txt +++ b/requirements/requirements_git.txt @@ -1,6 +1,6 @@ git+https://github.com/ansible/system-certifi.git@devel#egg=certifi # Remove pbr from requirements.in when moving ansible-runner to requirements.in git+https://github.com/ansible/ansible-runner.git@devel#egg=ansible-runner -django-ansible-base @ git+https://github.com/alancoding/django-ansible-base@advisory_lock#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac] +django-ansible-base @ git+https://github.com/ansible/django-ansible-base@devel#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac] awx-plugins-core @ git+https://git@github.com/ansible/awx-plugins.git@devel#egg=awx-plugins-core awx_plugins.interfaces @ git+https://github.com/ansible/awx_plugins.interfaces.git From 66ffbe1a685dff7b874738887c9ce04d3b671b66 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 18 Dec 2024 16:38:38 -0500 Subject: [PATCH 5/7] Re-run updater script --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 7d61cb24100e..0d8698338d6d 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -131,7 +131,7 @@ django==4.2.16 # django-polymorphic # django-solo # djangorestframework -# django-ansible-base @ git+https://github.com/alancoding/django-ansible-base@advisory_lock # git requirements installed separately +# django-ansible-base @ git+https://github.com/ansible/django-ansible-base@devel # git requirements installed separately # via -r /awx_devel/requirements/requirements_git.txt django-cors-headers==4.6.0 # via -r /awx_devel/requirements/requirements.in From 2915ee72e206300e6f30d23a9f88728da431f53a Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 8 Jan 2025 10:39:19 -0500 Subject: [PATCH 6/7] Remove comment, rebase issue --- requirements/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0d8698338d6d..8d0050fdcf12 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -454,7 +454,6 @@ setuptools-scm[toml]==8.1.0 six==1.17.0 # via # azure-core - # isodate # kubernetes # msrestazure # openshift From 7a6fe07591b32ebd0999d3e48951ebf5e083340a Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 8 Jan 2025 10:47:49 -0500 Subject: [PATCH 7/7] Move the import in new location --- awx/main/tasks/host_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/tasks/host_metrics.py b/awx/main/tasks/host_metrics.py index 9e163cfa73d9..999f4b63f1f3 100644 --- a/awx/main/tasks/host_metrics.py +++ b/awx/main/tasks/host_metrics.py @@ -11,7 +11,7 @@ from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly from awx.main.tasks.helpers import is_run_threshold_reached from awx.conf.license import get_license -from awx.main.utils.pglock import advisory_lock +from ansible_base.lib.utils.db import advisory_lock logger = logging.getLogger('awx.main.tasks.host_metrics')