catch harakiri graceful signal in middlware and log debug info

awx/main/apps.py

@@ -1,7 +1,10 @@
+import signal
+
 from django.apps import AppConfig
 from django.utils.translation import gettext_lazy as _
 from awx.main.utils.named_url_graph import _customize_graph, generate_graph
 from awx.conf import register, fields
+from awx.main.harakiri_middleware import HarakiriLoggerMiddleware
 
 
 class MainConfig(AppConfig):
@@ -36,5 +39,5 @@ def load_named_url_feature(self):
 
     def ready(self):
         super().ready()
-
         self.load_named_url_feature()
+        signal.signal(signal.SIGSYS, HarakiriLoggerMiddleware.handle_signal)

awx/main/harakiri_middleware.py

@@ -0,0 +1,28 @@
+import traceback
+import logging
+import uuid
+
+logger = logging.getLogger('awx.main.middleware')
+
+
+class HarakiriLoggerMiddleware:
+    transactions = {}
+
+    @classmethod
+    def handle_signal(cls, *args):
+        for t_id, request in HarakiriLoggerMiddleware.transactions.items():
+            logger.error(f"Catching harakiri graceful signal for {t_id} with method: {request.method} path: {request.path}")
+            logger.error(f"Received harakiri graceful signal while in stack: {''.join(traceback.format_stack()[-5:])}")
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        t_id = str(uuid.uuid4())
+        if "X-REQUEST-ID" in request.headers:
+            t_id = request.headers["X-REQUEST-ID"]
+        HarakiriLoggerMiddleware.transactions[t_id] = request
+        try:
+            return self.get_response(request)
+        finally:
+            HarakiriLoggerMiddleware.transactions.pop(t_id)

awx/settings/defaults.py

@@ -1016,6 +1016,7 @@
 
 MIDDLEWARE = [
     'django_guid.middleware.guid_middleware',
+    'awx.main.harakiri_middleware.HarakiriLoggerMiddleware',
     'awx.main.middleware.SettingsCacheMiddleware',
     'awx.main.middleware.TimingMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',

tools/ansible/roles/dockerfile/files/uwsgi.ini

@@ -10,6 +10,11 @@ master-fifo = /var/lib/awx/awxfifo
 max-requests = 1000
 buffer-size = 32768
 
+harakiri = 130
+harakiri-graceful-timeout = 120
+harakiri-graceful-signal = 31
+py-call-osafterfork = true
+
 if-env = UWSGI_MOUNT_PATH
 mount = %(_)=awx.wsgi:application
 endif =