From 9160d9127850e09497012c88ebfc13ed4c34569f Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 28 Mar 2019 17:11:03 -0400 Subject: [PATCH] clean up old usage of `idle_timeout` cmeyers and I looked at this and can't tell where/why/how you'd actually set this setting - it looks like really old ~2014-2015 Tower history that probably isn't actually in use --- awx/main/expect/isolated_manager.py | 23 +++++++++++++++++------ awx/main/tasks.py | 19 +------------------ 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/awx/main/expect/isolated_manager.py b/awx/main/expect/isolated_manager.py index e73c45d551..0830b390c7 100644 --- a/awx/main/expect/isolated_manager.py +++ b/awx/main/expect/isolated_manager.py @@ -29,17 +29,14 @@ def set_pythonpath(venv_libdir, env): class IsolatedManager(object): - def __init__(self, cancelled_callback=None, idle_timeout=None): + def __init__(self, cancelled_callback=None): """ :param cancelled_callback: a callable - which returns `True` or `False` - signifying if the job has been prematurely cancelled - :param idle_timeout a timeout (in seconds); if new output is not - sent to stdout in this interval, the process - will be terminated """ self.cancelled_callback = cancelled_callback - self.idle_timeout = idle_timeout or max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT) + self.idle_timeout = max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT) self.started_at = None def build_runner_params(self, hosts, verbosity=1): @@ -50,7 +47,21 @@ class IsolatedManager(object): set_pythonpath(os.path.join(settings.ANSIBLE_VENV_PATH, 'lib'), env) def finished_callback(runner_obj): - if runner_obj.status == 'failed': + if runner_obj.status == 'failed' and runner_obj.config.playbook != 'check_isolated.yml': + # failed for clean_isolated.yml just means the playbook hasn't + # exited on the isolated host + stdout = runner_obj.stdout.read() + playbook_logger.error(stdout) + elif runner_obj.status == 'timeout': + # this means that the default idle timeout of + # (2 * AWX_ISOLATED_CONNECTION_TIMEOUT) was exceeded + # (meaning, we tried to sync with an isolated node, and we got + # no new output for 2 * AWX_ISOLATED_CONNECTION_TIMEOUT seconds) + # this _usually_ means SSH key auth from the controller -> + # isolated didn't work, and ssh is hung waiting on interactive + # input e.g., + # + # awx@isolated's password: stdout = runner_obj.stdout.read() playbook_logger.error(stdout) else: diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 80f288ddc3..8eda2e9dbc 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -946,9 +946,6 @@ class BaseTask(object): def build_credentials_list(self, instance): return [] - def get_idle_timeout(self): - return None - def get_instance_timeout(self, instance): global_timeout_setting_name = instance._global_timeout_setting() if global_timeout_setting_name: @@ -1149,7 +1146,6 @@ class BaseTask(object): 'finished_callback': self.finished_callback, 'status_handler': self.status_handler, 'settings': { - 'idle_timeout': self.get_idle_timeout() or "", 'job_timeout': self.get_instance_timeout(self.instance), 'pexpect_timeout': getattr(settings, 'PEXPECT_TIMEOUT', 5), **process_isolation_params, @@ -1187,8 +1183,7 @@ class BaseTask(object): copy_tree(cwd, os.path.join(private_data_dir, 'project')) ansible_runner.utils.dump_artifacts(params) manager_instance = isolated_manager.IsolatedManager( - cancelled_callback=lambda: self.update_model(self.instance.pk).cancel_flag, - idle_timeout=self.get_idle_timeout(), + cancelled_callback=lambda: self.update_model(self.instance.pk).cancel_flag ) status, rc = manager_instance.run(self.instance, private_data_dir, @@ -1492,9 +1487,6 @@ class RunJob(BaseTask): def build_credentials_list(self, job): return job.credentials.all() - def get_idle_timeout(self): - return getattr(settings, 'JOB_RUN_IDLE_TIMEOUT', None) - def get_password_prompts(self, passwords={}): d = super(RunJob, self).get_password_prompts(passwords) d[r'Enter passphrase for .*:\s*?$'] = 'ssh_key_unlock' @@ -1768,9 +1760,6 @@ class RunProjectUpdate(BaseTask): d[r'^Are you sure you want to continue connecting \(yes/no\)\?\s*?$'] = 'yes' return d - def get_idle_timeout(self): - return getattr(settings, 'PROJECT_UPDATE_IDLE_TIMEOUT', None) - def _update_dependent_inventories(self, project_update, dependent_inventory_sources): scm_revision = project_update.project.scm_revision inv_update_class = InventoryUpdate._get_task_class() @@ -2131,9 +2120,6 @@ class RunInventoryUpdate(BaseTask): # All credentials not used by inventory source injector return inventory_update.get_extra_credentials() - def get_idle_timeout(self): - return getattr(settings, 'INVENTORY_UPDATE_IDLE_TIMEOUT', None) - def pre_run_hook(self, inventory_update): source_project = None if inventory_update.inventory_source: @@ -2321,9 +2307,6 @@ class RunAdHocCommand(BaseTask): def build_playbook_path_relative_to_cwd(self, job, private_data_dir): return None - def get_idle_timeout(self): - return getattr(settings, 'JOB_RUN_IDLE_TIMEOUT', None) - def get_password_prompts(self, passwords={}): d = super(RunAdHocCommand, self).get_password_prompts() d[r'Enter passphrase for .*:\s*?$'] = 'ssh_key_unlock'