diff --git a/awx/main/apps.py b/awx/main/apps.py index 099caea96a6b..2b618fddee5d 100644 --- a/awx/main/apps.py +++ b/awx/main/apps.py @@ -1,7 +1,10 @@ +import signal + from django.apps import AppConfig from django.utils.translation import gettext_lazy as _ from awx.main.utils.named_url_graph import _customize_graph, generate_graph from awx.conf import register, fields +from awx.main.harakiri_middleware import HarakiriLoggerMiddleware class MainConfig(AppConfig): @@ -36,5 +39,5 @@ def load_named_url_feature(self): def ready(self): super().ready() - self.load_named_url_feature() + signal.signal(signal.SIGSYS, HarakiriLoggerMiddleware.handle_signal) diff --git a/awx/main/harakiri_middleware.py b/awx/main/harakiri_middleware.py new file mode 100644 index 000000000000..1c1658f8e737 --- /dev/null +++ b/awx/main/harakiri_middleware.py @@ -0,0 +1,28 @@ +import traceback +import logging +import uuid + +logger = logging.getLogger('awx.main.middleware') + + +class HarakiriLoggerMiddleware: + transactions = {} + + @classmethod + def handle_signal(cls, *args): + for t_id, request in HarakiriLoggerMiddleware.transactions.items(): + logger.error(f"Catching harakiri graceful signal for {t_id} with method: {request.method} path: {request.path}") + logger.error(f"Received harakiri graceful signal while in stack: {''.join(traceback.format_stack()[-5:])}") + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + t_id = str(uuid.uuid4()) + if "X-REQUEST-ID" in request.headers: + t_id = request.headers["X-REQUEST-ID"] + HarakiriLoggerMiddleware.transactions[t_id] = request + try: + return self.get_response(request) + finally: + HarakiriLoggerMiddleware.transactions.pop(t_id) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 7f40dd976368..37b90b7d132c 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -1016,6 +1016,7 @@ MIDDLEWARE = [ 'django_guid.middleware.guid_middleware', + 'awx.main.harakiri_middleware.HarakiriLoggerMiddleware', 'awx.main.middleware.SettingsCacheMiddleware', 'awx.main.middleware.TimingMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', diff --git a/tools/ansible/roles/dockerfile/files/uwsgi.ini b/tools/ansible/roles/dockerfile/files/uwsgi.ini index bfd1f87f701b..43091b850932 100644 --- a/tools/ansible/roles/dockerfile/files/uwsgi.ini +++ b/tools/ansible/roles/dockerfile/files/uwsgi.ini @@ -10,6 +10,11 @@ master-fifo = /var/lib/awx/awxfifo max-requests = 1000 buffer-size = 32768 +harakiri = 130 +harakiri-graceful-timeout = 120 +harakiri-graceful-signal = 31 +py-call-osafterfork = true + if-env = UWSGI_MOUNT_PATH mount = %(_)=awx.wsgi:application endif =