setup playbook and heartbeat for isolated deployments

* Allow isolated_group_ use in setup playbook
* Tweaks to host/queue registration commands complementing setup
* Create isolated heartbeat task and check capacity
* Add content about isolated instances to acceptance docs

awx/main/tests/functional/models/test_unified_job.py

@@ -4,7 +4,7 @@ import pytest
 from django.contrib.contenttypes.models import ContentType
 
 # AWX
-from awx.main.models import UnifiedJobTemplate, Job, JobTemplate, WorkflowJobTemplate, Project
+from awx.main.models import UnifiedJobTemplate, Job, JobTemplate, WorkflowJobTemplate, Project, UnifiedJob
 
 
 @pytest.mark.django_db
@@ -65,3 +65,16 @@ class TestCreateUnifiedJob:
         assert second_job.inventory == job_with_links.inventory
         assert second_job.limit == 'my_server'
         assert net_credential in second_job.extra_credentials.all()
+
+
+@pytest.mark.django_db
+def test_lowest_running_id():
+    assert UnifiedJob.lowest_running_id() == 1
+    Job.objects.create(status='finished')
+    old_job = Job.objects.create(status='finished')
+    assert UnifiedJob.lowest_running_id() == old_job.id + 1
+    old_running_job = Job.objects.create(status='running')
+    Job.objects.create(status='running')
+    assert UnifiedJob.lowest_running_id() == old_running_job.id
+    Job.objects.create(status='finished')
+    assert UnifiedJob.lowest_running_id() == old_running_job.id

awx/main/tests/functional/test_tasks.py

@@ -2,8 +2,17 @@ import pytest
 import mock
 import os
 
-from awx.main.tasks import RunProjectUpdate, RunInventoryUpdate
-from awx.main.models import ProjectUpdate, InventoryUpdate, InventorySource
+from django.utils.timezone import now, timedelta
+
+from awx.main.tasks import (
+    RunProjectUpdate, RunInventoryUpdate,
+    tower_isolated_heartbeat,
+    isolated_manager
+)
+from awx.main.models import (
+    ProjectUpdate, InventoryUpdate, InventorySource,
+    Instance, InstanceGroup
+)
 
 
 @pytest.fixture
@@ -73,3 +82,57 @@ class TestDependentInventoryUpdate:
             # Verify that it bails after 1st update, detecting a cancel
             assert is2.inventory_updates.count() == 0
             iu_run_mock.assert_called_once()
+
+
+
+class MockSettings:
+    AWX_ISOLATED_PERIODIC_CHECK = 60
+    CLUSTER_HOST_ID = 'tower_1'
+
+
+@pytest.mark.django_db
+class TestIsolatedManagementTask:
+
+    @pytest.fixture
+    def control_group(self):
+        return InstanceGroup.objects.create(name='alpha')
+
+    @pytest.fixture
+    def control_instance(self, control_group):
+        return control_group.instances.create(hostname='tower_1')
+
+    @pytest.fixture
+    def needs_updating(self, control_group):
+        ig = InstanceGroup.objects.create(name='thepentagon', controller=control_group)
+        inst = ig.instances.create(
+            hostname='isolated', capacity=103)
+        inst.last_isolated_check=now() - timedelta(seconds=MockSettings.AWX_ISOLATED_PERIODIC_CHECK)
+        inst.save()
+        return ig
+
+    @pytest.fixture
+    def just_updated(self, control_group):
+        ig = InstanceGroup.objects.create(name='thepentagon', controller=control_group)
+        inst = ig.instances.create(
+            hostname='isolated', capacity=103)
+        inst.last_isolated_check=now()
+        inst.save()
+        return inst
+
+    def test_takes_action(self, control_instance, needs_updating):
+        with mock.patch('awx.main.tasks.settings', MockSettings()):
+            with mock.patch.object(isolated_manager.IsolatedManager, 'health_check') as check_mock:
+                check_mock.return_value = 98
+                tower_isolated_heartbeat()
+        iso_instance = Instance.objects.get(hostname='isolated')
+        check_mock.assert_called_once_with(iso_instance, cutoff_pk=mock.ANY)
+        assert iso_instance.capacity == 98
+
+    def test_does_not_take_action(self, control_instance, just_updated):
+        with mock.patch('awx.main.tasks.settings', MockSettings()):
+            with mock.patch.object(isolated_manager.IsolatedManager, 'health_check') as check_mock:
+                check_mock.return_value = 98
+                tower_isolated_heartbeat()
+        iso_instance = Instance.objects.get(hostname='isolated')
+        check_mock.assert_not_called()
+        assert iso_instance.capacity == 103