properly handle isolated cancellation

awx/main/expect/isolated_manager.py

@@ -21,8 +21,7 @@ playbook_logger = logging.getLogger('awx.isolated.manager.playbooks')
 class IsolatedManager(object):
 
     def __init__(self, env, cancelled_callback=None, job_timeout=0,
-                 idle_timeout=None, extra_update_fields=None,
+                 idle_timeout=None):
-                 pexpect_timeout=5, proot_cmd='bwrap'):
         """
         :param env:                 a dict containing environment variables for the
                                     subprocess, ala `os.environ`
@@ -34,20 +33,11 @@ class IsolatedManager(object):
         :param idle_timeout         a timeout (in seconds); if new output is not
                                     sent to stdout in this interval, the process
                                     will be terminated
-        :param extra_update_fields: a dict used to specify DB fields which should
-                                    be updated on the underlying model
-                                    object after execution completes
-        :param pexpect_timeout      a timeout (in seconds) to wait on
-                                    `pexpect.spawn().expect()` calls
-        :param proot_cmd            the command used to isolate processes, `bwrap`
         """
         self.management_env = self._base_management_env()
         self.cancelled_callback = cancelled_callback
         self.job_timeout = job_timeout
         self.idle_timeout = idle_timeout
-        self.extra_update_fields = extra_update_fields
-        self.pexpect_timeout = pexpect_timeout
-        self.proot_cmd = proot_cmd
         self.started_at = None
 
     @staticmethod
@@ -216,13 +206,6 @@ class IsolatedManager(object):
         while status == 'failed':
             if job_timeout != 0:
                 remaining = max(0, job_timeout - (time.time() - self.started_at))
-                if remaining == 0:
-                    # if it takes longer than $REMAINING_JOB_TIMEOUT to retrieve
-                    # job artifacts from the host, consider the job failed
-                    if isinstance(self.extra_update_fields, dict):
-                        self.extra_update_fields['job_explanation'] = "Job terminated due to timeout"
-                    status = 'failed'
-                    break
 
             canceled = self.cancelled_callback() if self.cancelled_callback else False
             if not canceled and time.time() - last_check < interval:
@@ -230,6 +213,9 @@ class IsolatedManager(object):
                 time.sleep(1)
                 continue
 
+            if canceled:
+                logger.warning('Isolated job {} was manually cancelled.'.format(self.instance.id))
+
             buff = StringIO()
             logger.debug('Checking on isolated job {} with `check_isolated.yml`.'.format(self.instance.id))
             status, rc = IsolatedManager.run_pexpect(
@@ -238,7 +224,7 @@ class IsolatedManager(object):
                 idle_timeout=remaining,
                 job_timeout=remaining,
                 pexpect_timeout=5,
-                proot_cmd=self.proot_cmd
+                proot_cmd='bwrap'
             )
             output = buff.getvalue().encode('utf-8')
             playbook_logger.info('Isolated job {} check:\n{}'.format(self.instance.id, output))

awx/main/tasks.py

@@ -1139,11 +1139,6 @@ class BaseTask(object):
             password_prompts = self.get_password_prompts(passwords)
             expect_passwords = self.create_expect_passwords_data_struct(password_prompts, passwords)
 
-            # TODO: Probably remove this when cleaning up isolated path
-            _kw = dict(
-                extra_update_fields=extra_update_fields,
-                proot_cmd=getattr(settings, 'AWX_PROOT_CMD', 'bwrap'),
-            )
             self.instance = self.update_model(self.instance.pk, output_replacements=output_replacements)
 
             params = {
@@ -1196,7 +1191,12 @@ class BaseTask(object):
                 )
                 copy_tree(cwd, os.path.join(private_data_dir, 'project'))
                 ansible_runner.utils.dump_artifacts(params)
-                manager_instance = isolated_manager.IsolatedManager(env, **_kw)
+                manager_instance = isolated_manager.IsolatedManager(
+                    env,
+                    cancelled_callback=lambda: self.update_model(self.instance.pk).cancel_flag,
+                    job_timeout=self.get_instance_timeout(self.instance),
+                    idle_timeout=self.get_idle_timeout(),
+                )
                 status, rc = manager_instance.run(self.instance,
                                                   private_data_dir,
                                                   params.get('playbook'),