External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-15 03:40:42 -03:30
Code Issues Packages Projects Releases Wiki Activity

properly handle isolated cancellation

Browse Source
This commit is contained in:
Ryan Petrello 2019-03-27 11:44:26 -04:00
parent 055e7b4974
commit a983d4bc1f
No known key found for this signature in database
GPG Key ID: F2AA5F2122351777
2 changed files with 11 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
awx/main/expect/isolated_manager.py
View File

@ -21,8 +21,7 @@ playbook_logger = logging.getLogger('awx.isolated.manager.playbooks')
class IsolatedManager(object):
def __init__(self, env, cancelled_callback=None, job_timeout=0,
idle_timeout=None, extra_update_fields=None,
pexpect_timeout=5, proot_cmd='bwrap'):
idle_timeout=None):
"""
:param env: a dict containing environment variables for the
subprocess, ala `os.environ`
@ -34,20 +33,11 @@ class IsolatedManager(object):
:param idle_timeout a timeout (in seconds); if new output is not
sent to stdout in this interval, the process
will be terminated
:param extra_update_fields: a dict used to specify DB fields which should
be updated on the underlying model
object after execution completes
:param pexpect_timeout a timeout (in seconds) to wait on
`pexpect.spawn().expect()` calls
:param proot_cmd the command used to isolate processes, `bwrap`
"""
self.management_env = self._base_management_env()
self.cancelled_callback = cancelled_callback
self.job_timeout = job_timeout
self.idle_timeout = idle_timeout
self.extra_update_fields = extra_update_fields
self.pexpect_timeout = pexpect_timeout
self.proot_cmd = proot_cmd
self.started_at = None
@staticmethod
@ -216,13 +206,6 @@ class IsolatedManager(object):
while status == 'failed':
if job_timeout != 0:
remaining = max(0, job_timeout - (time.time() - self.started_at))
if remaining == 0:
# if it takes longer than $REMAINING_JOB_TIMEOUT to retrieve
# job artifacts from the host, consider the job failed
if isinstance(self.extra_update_fields, dict):
self.extra_update_fields['job_explanation'] = "Job terminated due to timeout"
status = 'failed'
break
canceled = self.cancelled_callback() if self.cancelled_callback else False
if not canceled and time.time() - last_check < interval:
@ -230,6 +213,9 @@ class IsolatedManager(object):
time.sleep(1)
continue
if canceled:
logger.warning('Isolated job {} was manually cancelled.'.format(self.instance.id))
buff = StringIO()
logger.debug('Checking on isolated job {} with `check_isolated.yml`.'.format(self.instance.id))
status, rc = IsolatedManager.run_pexpect(
@ -238,7 +224,7 @@ class IsolatedManager(object):
idle_timeout=remaining,
job_timeout=remaining,
pexpect_timeout=5,
proot_cmd=self.proot_cmd
proot_cmd='bwrap'
)
output = buff.getvalue().encode('utf-8')
playbook_logger.info('Isolated job {} check:\n{}'.format(self.instance.id, output))

12
awx/main/tasks.py
View File

@ -1139,11 +1139,6 @@ class BaseTask(object):
password_prompts = self.get_password_prompts(passwords)
expect_passwords = self.create_expect_passwords_data_struct(password_prompts, passwords)
# TODO: Probably remove this when cleaning up isolated path
_kw = dict(
extra_update_fields=extra_update_fields,
proot_cmd=getattr(settings, 'AWX_PROOT_CMD', 'bwrap'),
)
self.instance = self.update_model(self.instance.pk, output_replacements=output_replacements)
params = {
@ -1196,7 +1191,12 @@ class BaseTask(object):
)
copy_tree(cwd, os.path.join(private_data_dir, 'project'))
ansible_runner.utils.dump_artifacts(params)
manager_instance = isolated_manager.IsolatedManager(env, **_kw)
manager_instance = isolated_manager.IsolatedManager(
env,
cancelled_callback=lambda: self.update_model(self.instance.pk).cancel_flag,
job_timeout=self.get_instance_timeout(self.instance),
idle_timeout=self.get_idle_timeout(),
)
status, rc = manager_instance.run(self.instance,
private_data_dir,
params.get('playbook'),