From bc0626fe0f9ef20713ac5733610b78e0794b70b1 Mon Sep 17 00:00:00 2001 From: Elijah DeLee Date: Fri, 16 Aug 2024 11:16:14 -0400 Subject: [PATCH] catch harakiri graceful signal in middlware and log debug info --- awx/main/apps.py | 5 +++- awx/main/harakiri_middleware.py | 28 +++++++++++++++++++ awx/settings/defaults.py | 1 + .../ansible/roles/dockerfile/files/uwsgi.ini | 5 ++++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 awx/main/harakiri_middleware.py diff --git a/awx/main/apps.py b/awx/main/apps.py index 099caea96a6b..2b618fddee5d 100644 --- a/awx/main/apps.py +++ b/awx/main/apps.py @@ -1,7 +1,10 @@ +import signal + from django.apps import AppConfig from django.utils.translation import gettext_lazy as _ from awx.main.utils.named_url_graph import _customize_graph, generate_graph from awx.conf import register, fields +from awx.main.harakiri_middleware import HarakiriLoggerMiddleware class MainConfig(AppConfig): @@ -36,5 +39,5 @@ def load_named_url_feature(self): def ready(self): super().ready() - self.load_named_url_feature() + signal.signal(signal.SIGSYS, HarakiriLoggerMiddleware.handle_signal) diff --git a/awx/main/harakiri_middleware.py b/awx/main/harakiri_middleware.py new file mode 100644 index 000000000000..1c1658f8e737 --- /dev/null +++ b/awx/main/harakiri_middleware.py @@ -0,0 +1,28 @@ +import traceback +import logging +import uuid + +logger = logging.getLogger('awx.main.middleware') + + +class HarakiriLoggerMiddleware: + transactions = {} + + @classmethod + def handle_signal(cls, *args): + for t_id, request in HarakiriLoggerMiddleware.transactions.items(): + logger.error(f"Catching harakiri graceful signal for {t_id} with method: {request.method} path: {request.path}") + logger.error(f"Received harakiri graceful signal while in stack: {''.join(traceback.format_stack()[-5:])}") + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + t_id = str(uuid.uuid4()) + if "X-REQUEST-ID" in request.headers: + t_id = request.headers["X-REQUEST-ID"] + HarakiriLoggerMiddleware.transactions[t_id] = request + try: + return self.get_response(request) + finally: + HarakiriLoggerMiddleware.transactions.pop(t_id) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 7f40dd976368..37b90b7d132c 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -1016,6 +1016,7 @@ MIDDLEWARE = [ 'django_guid.middleware.guid_middleware', + 'awx.main.harakiri_middleware.HarakiriLoggerMiddleware', 'awx.main.middleware.SettingsCacheMiddleware', 'awx.main.middleware.TimingMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', diff --git a/tools/ansible/roles/dockerfile/files/uwsgi.ini b/tools/ansible/roles/dockerfile/files/uwsgi.ini index bfd1f87f701b..43091b850932 100644 --- a/tools/ansible/roles/dockerfile/files/uwsgi.ini +++ b/tools/ansible/roles/dockerfile/files/uwsgi.ini @@ -10,6 +10,11 @@ master-fifo = /var/lib/awx/awxfifo max-requests = 1000 buffer-size = 32768 +harakiri = 130 +harakiri-graceful-timeout = 120 +harakiri-graceful-signal = 31 +py-call-osafterfork = true + if-env = UWSGI_MOUNT_PATH mount = %(_)=awx.wsgi:application endif =