fix a few isolated dev issues

the main goal of this change is to make `make docker-isolated` work out
of the box

- specify the proper version for awx-expect --version
- update some deprecated playbook bits
- change the isolated container to privileged so bwrap will work
- fix awx-manage test_isolated_connection
- expedite the first isolated heartbeat so you don't have to wait 10m;
  this is accomplished by _not_ setting Instance.last_isolated_check to
  now() at insertion time (which causes the next check not to happen for
  10 minutes)
- fix a bug that caused isolated node execution to fail when bwrap was
  enabled

see: https://github.com/ansible/tower/issues/2150

This reverts commit 9863fe71dc.

awx/main/expect/isolated_manager.py

@@ -434,6 +434,7 @@ class IsolatedManager(object):
                 task_result = {}
             if 'capacity_cpu' in task_result and 'capacity_mem' in task_result:
                 cls.update_capacity(instance, task_result, awx_application_version)
+                logger.debug('Isolated instance {} successful heartbeat'.format(instance.hostname))
             elif instance.capacity == 0:
                 logger.debug('Isolated instance {} previously marked as lost, could not re-join.'.format(
                     instance.hostname))

awx/main/management/commands/test_isolated_connection.py

@@ -3,7 +3,6 @@ import shutil
 import subprocess
 import sys
 import tempfile
-from optparse import make_option
 
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
@@ -15,10 +14,9 @@ class Command(BaseCommand):
     """Tests SSH connectivity between a controller and target isolated node"""
     help = 'Tests SSH connectivity between a controller and target isolated node'
 
-    option_list = BaseCommand.option_list + (
-        make_option('--hostname', dest='hostname', type='string',
-                    help='Hostname of an isolated node'),
-    )
+    def add_arguments(self, parser):
+        parser.add_argument('--hostname', dest='hostname', type=str,
+                            help='Hostname of an isolated node')
 
     def handle(self, *args, **options):
         hostname = options.get('hostname')
@@ -30,7 +28,7 @@ class Command(BaseCommand):
             args = [
                 'ansible', 'all', '-i', '{},'.format(hostname), '-u',
                 settings.AWX_ISOLATED_USERNAME, '-T5', '-m', 'shell',
-                '-a', 'hostname', '-vvv'
+                '-a', 'awx-expect -h', '-vvv'
             ]
             if all([
                 getattr(settings, 'AWX_ISOLATED_KEY_GENERATION', False) is True,

awx/main/migrations/0006_v320_release.py

@@ -484,7 +484,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='instance',
             name='last_isolated_check',
-            field=models.DateTimeField(auto_now_add=True, null=True),
+            field=models.DateTimeField(editable=False, null=True),
         ),
         # Migrations that don't change db schema but simply to make Django ORM happy.
         # e.g. Choice updates, help_text updates, etc.

awx/main/models/ha.py

@@ -50,7 +50,6 @@ class Instance(BaseModel):
     last_isolated_check = models.DateTimeField(
         null=True,
         editable=False,
-        auto_now_add=True
     )
     version = models.CharField(max_length=24, blank=True)
     capacity = models.PositiveIntegerField(

awx/main/tasks.py

@@ -226,6 +226,7 @@ def handle_ha_toplogy_worker_ready(sender, **kwargs):
     # Expedite the first hearbeat run so a node comes online quickly.
     cluster_node_heartbeat.apply([])
     apply_cluster_membership_policies.apply([])
+    awx_isolated_heartbeat.apply([])
 
 
 @celeryd_after_setup.connect
@@ -380,7 +381,11 @@ def awx_isolated_heartbeat(self):
     accept_before = nowtime - timedelta(seconds=(poll_interval - 10))
     isolated_instance_qs = Instance.objects.filter(
         rampart_groups__controller__instances__hostname=local_hostname,
+    )
+    isolated_instance_qs = isolated_instance_qs.filter(
         last_isolated_check__lt=accept_before
+    ) | isolated_instance_qs.filter(
+        last_isolated_check=None
     )
     # Fast pass of isolated instances, claiming the nodes to update
     with transaction.atomic():
@@ -883,6 +888,7 @@ class BaseTask(Task):
         stdout_handle = None
 
         try:
+            kwargs['isolated'] = instance.is_isolated()
             self.pre_run_hook(instance, **kwargs)
             if instance.cancel_flag:
                 instance = self.update_model(instance.pk, status='canceled')