setup playbook and heartbeat for isolated deployments

* Allow isolated_group_ use in setup playbook
* Tweaks to host/queue registration commands complementing setup
* Create isolated heartbeat task and check capacity
* Add content about isolated instances to acceptance docs

awx/main/tasks.py

@@ -19,7 +19,6 @@ import traceback
 import urlparse
 import uuid
 from distutils.version import LooseVersion as Version
-from datetime import timedelta
 import yaml
 import fcntl
 try:
@@ -34,7 +33,7 @@ from celery.signals import celeryd_init, worker_process_init
 # Django
 from django.conf import settings
 from django.db import transaction, DatabaseError, IntegrityError
-from django.utils.timezone import now
+from django.utils.timezone import now, timedelta
 from django.utils.encoding import smart_str
 from django.core.mail import send_mail
 from django.contrib.auth.models import User
@@ -197,6 +196,33 @@ def cluster_node_heartbeat(self):
             stop_local_services(['uwsgi', 'celery', 'beat', 'callback', 'fact'])
 
 
+@task(bind=True)
+def tower_isolated_heartbeat(self):
+    local_hostname = settings.CLUSTER_HOST_ID
+    logger.debug("Controlling node checking for any isolated management tasks.")
+    poll_interval = settings.AWX_ISOLATED_PERIODIC_CHECK
+    # Add in some task buffer time
+    nowtime = now()
+    accept_before = nowtime - timedelta(seconds=(poll_interval - 10))
+    isolated_instance_qs = Instance.objects.filter(
+        rampart_groups__controller__instances__hostname=local_hostname,
+        last_isolated_check__lt=accept_before
+    )
+    # Fast pass of isolated instances, claiming the nodes to update
+    with transaction.atomic():
+        for isolated_instance in isolated_instance_qs:
+            isolated_instance.last_isolated_check = nowtime
+            isolated_instance.save(update_fields=['last_isolated_check'])
+    # Find the oldest job in the system and pass that to the cleanup
+    if not isolated_instance_qs:
+        return
+    cutoff_pk = UnifiedJob.lowest_running_id()
+    # Slow pass looping over isolated IGs and their isolated instances
+    for isolated_instance in isolated_instance_qs:
+        logger.debug("Managing isolated instance {}.".format(isolated_instance.hostname))
+        isolated_instance.capacity = isolated_manager.IsolatedManager.health_check(isolated_instance, cutoff_pk=cutoff_pk)
+        isolated_instance.save(update_fields=['capacity'])
+
 
 @task(bind=True, queue='tower')
 def tower_periodic_scheduler(self):