add an AWX_ISOLATED_VERBOSITY setting for debugging isolated connections

awx/main/conf.py

@@ -197,6 +197,18 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'AWX_ISOLATED_VERBOSITY',
+    field_class=fields.IntegerField,
+    min_value=0,
+    max_value=5,
+    label=_('Verbosity level for isolated node management tasks'),
+    help_text=_('This can be raised to aid in debugging connection issues for isolated task execution'),
+    category=_('Jobs'),
+    category_slug='jobs',
+    default=0
+)
+
 register(
     'AWX_ISOLATED_CHECK_INTERVAL',
     field_class=fields.IntegerField,

awx/main/expect/isolated_manager.py

@@ -101,6 +101,8 @@ class IsolatedManager(object):
         ]
         if extra_vars:
             args.extend(['-e', json.dumps(extra_vars)])
+        if settings.AWX_ISOLATED_VERBOSITY:
+            args.append('-%s' % ('v' * min(5, settings.AWX_ISOLATED_VERBOSITY)))
         return args
 
     @staticmethod
@@ -420,16 +422,18 @@ class IsolatedManager(object):
                 idle_timeout=timeout, job_timeout=timeout,
                 pexpect_timeout=5
             )
+            heartbeat_stdout = buff.getvalue().encode('utf-8')
+            buff.close()
 
             for instance in instance_qs:
-                output = buff.getvalue()
+                output = heartbeat_stdout
+                task_result = {}
                 try:
                     with open(os.path.join(facts_path, instance.hostname), 'r') as facts_data:
                         output = facts_data.read()
                     task_result = json.loads(output)
                 except Exception:
                     logger.exception('Failed to read status from isolated instances, output:\n {}'.format(output))
-                    return
                 if 'awx_capacity_cpu' in task_result and 'awx_capacity_mem' in task_result:
                     task_result = {
                         'capacity_cpu': task_result['awx_capacity_cpu'],

awx/settings/defaults.py

@@ -430,6 +430,9 @@ AWX_ISOLATED_CONNECTION_TIMEOUT = 10
 # The time (in seconds) between the periodic isolated heartbeat status check
 AWX_ISOLATED_PERIODIC_CHECK = 600
 
+# Verbosity level for isolated node management tasks
+AWX_ISOLATED_VERBOSITY = 0
+
 # Memcached django cache configuration
 # CACHES = {
 #     'default': {