From b01d204137bd66326db4005d30463681fe947a6c Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Thu, 3 Sep 2020 13:02:10 -0400
Subject: [PATCH] if redis is unreachable, set instance capacity to zero

---
 awx/main/models/ha.py                  |  9 +++++++++
 awx/main/tests/functional/test_jobs.py | 17 ++++++++++++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/awx/main/models/ha.py b/awx/main/models/ha.py
index ac7df97e10..4b6d8926e9 100644
--- a/awx/main/models/ha.py
+++ b/awx/main/models/ha.py
@@ -12,6 +12,7 @@ from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
 from django.utils.timezone import now, timedelta
 
+import redis
 from solo.models import SingletonModel
 
 from awx import __version__ as awx_application_version
@@ -152,6 +153,14 @@ class Instance(HasPolicyEditsMixin, BaseModel):
             self.capacity = get_system_task_capacity(self.capacity_adjustment)
         else:
             self.capacity = 0
+
+        try:
+            # if redis is down for some reason, that means we can't persist
+            # playbook event data; we should consider this a zero capacity event
+            redis.Redis.from_url(settings.BROKER_URL).ping()
+        except redis.ConnectionError:
+            self.capacity = 0
+
         self.cpu = cpu[0]
         self.memory = mem[0]
         self.cpu_capacity = cpu[1]
diff --git a/awx/main/tests/functional/test_jobs.py b/awx/main/tests/functional/test_jobs.py
index 2bc10fa0df..b4754a6803 100644
--- a/awx/main/tests/functional/test_jobs.py
+++ b/awx/main/tests/functional/test_jobs.py
@@ -1,3 +1,4 @@
+import redis
 import pytest
 from unittest import mock
 import json
@@ -25,7 +26,8 @@ def test_orphan_unified_job_creation(instance, inventory):
 @mock.patch('awx.main.utils.common.get_mem_capacity', lambda: (8000,62))
 def test_job_capacity_and_with_inactive_node():
     i = Instance.objects.create(hostname='test-1')
-    i.refresh_capacity()
+    with mock.patch.object(redis.client.Redis, 'ping', lambda self: True):
+        i.refresh_capacity()
     assert i.capacity == 62
     i.enabled = False
     i.save()
@@ -35,6 +37,19 @@ def test_job_capacity_and_with_inactive_node():
         assert i.capacity == 0
 
 
+@pytest.mark.django_db
+@mock.patch('awx.main.utils.common.get_cpu_capacity', lambda: (2,8))
+@mock.patch('awx.main.utils.common.get_mem_capacity', lambda: (8000,62))
+def test_job_capacity_with_redis_disabled():
+    i = Instance.objects.create(hostname='test-1')
+
+    def _raise(self):
+        raise redis.ConnectionError()
+    with mock.patch.object(redis.client.Redis, 'ping', _raise):
+        i.refresh_capacity()
+    assert i.capacity == 0
+
+
 @pytest.mark.django_db
 def test_job_type_name():
     job = Job.objects.create()