Support expiring of capacity if a node is down

For a certain amount of time

awx/main/managers.py

@@ -2,8 +2,10 @@
 # All Rights Reserved.
 
 import sys
+from datetime import timedelta
 
 from django.db import models
+from django.utils.timezone import now
 from django.db.models import Sum
 from django.conf import settings
 
@@ -42,7 +44,9 @@ class InstanceManager(models.Manager):
         return self.all().count()
 
     def total_capacity(self):
-        return self.aggregate(total_capacity=Sum('capacity'))['total_capacity']
+        sumval = self.filter(modified__gte=now()-timedelta(seconds=settings.AWX_ACTIVE_NODE_TIME)) \
+                     .aggregate(total_capacity=Sum('capacity'))['total_capacity']
+        return max(50, sumval)
 
     def my_role(self):
         # NOTE: TODO: Likely to repurpose this once standalone ramparts are a thing

awx/main/tests/functional/conftest.py

@@ -153,7 +153,7 @@ def user_project(user):
 
 @pytest.fixture
 def instance(settings):
-    return Instance.objects.create(uuid=settings.SYSTEM_UUID, hostname="instance.example.org")
+    return Instance.objects.create(uuid=settings.SYSTEM_UUID, hostname="instance.example.org", capacity=100)
 
 @pytest.fixture
 def organization(instance):

awx/main/tests/functional/test_jobs.py

@@ -1,5 +1,10 @@
-from awx.main.models import Job
+from awx.main.models import Job, Instance
 
+from django.utils import timezone
+from django.conf import settings
+from django.test.utils import override_settings
+
+from datetime import timedelta
 import pytest
 
 @pytest.mark.django_db
@@ -9,3 +14,12 @@ def test_orphan_unified_job_creation(instance, inventory):
     assert job2.job_template is None
     assert job2.inventory == inventory
     assert job2.name == 'hi world'
+
+@pytest.mark.django_db
+def test_job_capacity_and_with_inactive_node():
+    i = Instance.objects.create(hostname='test-1', capacity=50)
+    assert Instance.objects.total_capacity() == 50
+    i2 = Instance.objects.create(hostname='test-2', capacity=50)
+    assert Instance.objects.total_capacity() == 100
+    with override_settings(AWX_ACTIVE_NODE_TIME=0):
+        assert Instance.objects.total_capacity() < 100

awx/settings/defaults.py

@@ -549,6 +549,9 @@ AWX_PROOT_BASE_PATH = "/tmp"
 # Note: This setting may be overridden by database settings.
 AWX_ANSIBLE_CALLBACK_PLUGINS = ""
 
+# Time at which an HA node is considered active
+AWX_ACTIVE_NODE_TIME = 7200
+
 # Enable Pendo on the UI, possible values are 'off', 'anonymous', and 'detailed'
 # Note: This setting may be overridden by database settings.
 PENDO_TRACKING_STATE = "off"