Merge pull request #6360 from chrismeyersfsu/log_notification_failures

log when notifications fail to send

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

awx/main/tasks.py

@@ -609,7 +609,7 @@ def handle_success_and_failure_notifications(job_id):
     while retries < 5:
         if uj.finished:
             uj.send_notification_templates('succeeded' if uj.status == 'successful' else 'failed')
-            break
+            return
         else:
             # wait a few seconds to avoid a race where the
             # events are persisted _before_ the UJ.status
@@ -618,6 +618,8 @@ def handle_success_and_failure_notifications(job_id):
             time.sleep(1)
             uj = UnifiedJob.objects.get(pk=job_id)
 
+    logger.warn(f"Failed to even try to send notifications for job '{uj}' due to job not being in finished state.")
+
 
 @task(queue=get_local_queuename)
 def update_inventory_computed_fields(inventory_id):

awx/main/tests/unit/test_tasks.py

@@ -2381,3 +2381,23 @@ def test_managed_injector_redaction(injector_cls):
             if secret_field_name in template:
                 env[env_name] = 'very_secret_value'
     assert 'very_secret_value' not in str(build_safe_env(env))
+
+
+@mock.patch('logging.getLogger')
+def test_notification_job_not_finished(logging_getLogger, mocker):
+    uj = mocker.MagicMock()
+    uj.finished = False
+    logger = mocker.Mock()
+    logging_getLogger.return_value = logger
+
+    with mocker.patch('awx.main.models.UnifiedJob.objects.get', uj):
+        tasks.handle_success_and_failure_notifications(1)
+        assert logger.warn.called_with(f"Failed to even try to send notifications for job '{uj}' due to job not being in finished state.")
+
+
+def test_notification_job_finished(mocker):
+    uj = mocker.MagicMock(send_notification_templates=mocker.MagicMock(), finished=True)
+
+    with mocker.patch('awx.main.models.UnifiedJob.objects.get', mocker.MagicMock(return_value=uj)):
+        tasks.handle_success_and_failure_notifications(1)
+        uj.send_notification_templates.assert_called()