Give specific messages if job was killed due to SIGTERM or SIGKILL (#12435)

* Reap jobs on dispatcher startup to increase clarity, replace existing reaping logic

* Exit jobs if receiving SIGTERM signal

* Fix unwanted reaping on shutdown, let subprocess close out

* Add some sanity tests for signal module

* Add a log for an unhandled dispatcher error

* Refine wording of error messages

Co-authored-by: Elijah DeLee <kdelee@redhat.com>

awx/main/tasks/signals.py

@@ -0,0 +1,63 @@
+import signal
+import functools
+import logging
+
+
+logger = logging.getLogger('awx.main.tasks.signals')
+
+
+__all__ = ['with_signal_handling', 'signal_callback']
+
+
+class SignalState:
+    def reset(self):
+        self.sigterm_flag = False
+        self.is_active = False
+        self.original_sigterm = None
+        self.original_sigint = None
+
+    def __init__(self):
+        self.reset()
+
+    def set_flag(self, *args):
+        """Method to pass into the python signal.signal method to receive signals"""
+        self.sigterm_flag = True
+
+    def connect_signals(self):
+        self.original_sigterm = signal.getsignal(signal.SIGTERM)
+        self.original_sigint = signal.getsignal(signal.SIGINT)
+        signal.signal(signal.SIGTERM, self.set_flag)
+        signal.signal(signal.SIGINT, self.set_flag)
+        self.is_active = True
+
+    def restore_signals(self):
+        signal.signal(signal.SIGTERM, self.original_sigterm)
+        signal.signal(signal.SIGINT, self.original_sigint)
+        self.reset()
+
+
+signal_state = SignalState()
+
+
+def signal_callback():
+    return signal_state.sigterm_flag
+
+
+def with_signal_handling(f):
+    """
+    Change signal handling to make signal_callback return True in event of SIGTERM or SIGINT.
+    """
+
+    @functools.wraps(f)
+    def _wrapped(*args, **kwargs):
+        try:
+            this_is_outermost_caller = False
+            if not signal_state.is_active:
+                signal_state.connect_signals()
+                this_is_outermost_caller = True
+            return f(*args, **kwargs)
+        finally:
+            if this_is_outermost_caller:
+                signal_state.restore_signals()
+
+    return _wrapped