Give specific messages if job was killed due to SIGTERM or SIGKILL (#12435)

* Reap jobs on dispatcher startup to increase clarity, replace existing reaping logic

* Exit jobs if receiving SIGTERM signal

* Fix unwanted reaping on shutdown, let subprocess close out

* Add some sanity tests for signal module

* Add a log for an unhandled dispatcher error

* Refine wording of error messages

Co-authored-by: Elijah DeLee <kdelee@redhat.com>

awx/main/tests/unit/tasks/test_signals.py

@@ -0,0 +1,50 @@
+import signal
+
+from awx.main.tasks.signals import signal_state, signal_callback, with_signal_handling
+
+
+def test_outer_inner_signal_handling():
+    """
+    Even if the flag is set in the outer context, its value should persist in the inner context
+    """
+
+    @with_signal_handling
+    def f2():
+        assert signal_callback()
+
+    @with_signal_handling
+    def f1():
+        assert signal_callback() is False
+        signal_state.set_flag()
+        assert signal_callback()
+        f2()
+
+    original_sigterm = signal.getsignal(signal.SIGTERM)
+    assert signal_callback() is False
+    f1()
+    assert signal_callback() is False
+    assert signal.getsignal(signal.SIGTERM) is original_sigterm
+
+
+def test_inner_outer_signal_handling():
+    """
+    Even if the flag is set in the inner context, its value should persist in the outer context
+    """
+
+    @with_signal_handling
+    def f2():
+        assert signal_callback() is False
+        signal_state.set_flag()
+        assert signal_callback()
+
+    @with_signal_handling
+    def f1():
+        assert signal_callback() is False
+        f2()
+        assert signal_callback()
+
+    original_sigterm = signal.getsignal(signal.SIGTERM)
+    assert signal_callback() is False
+    f1()
+    assert signal_callback() is False
+    assert signal.getsignal(signal.SIGTERM) is original_sigterm