External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-19 18:07:33 -02:30
Code Issues Packages Projects Releases Wiki Activity

remove safe_args and add status_handler

Browse Source 
* safe_args no longer makes sense. We have moved extra_vars to a file
and thus do not pass sensitive content on the cmdline
This commit is contained in:
chris meyers
2019-03-08 13:48:50 -05:00
parent 602ef9750f
commit 827ad0fa75
3 changed files with 89 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
awx/main/models/credential/__init__.py
View File

@@ -606,7 +606,7 @@ class CredentialType(CommonModelNameNotUnique):
match = cls.objects.filter(**requirements)[:1].get() match = cls.objects.filter(**requirements)[:1].get()
return match return match
def inject_credential(self, credential, env, safe_env, args, safe_args, private_data_dir): def inject_credential(self, credential, env, safe_env, args, private_data_dir):
""" """
Inject credential data into the environment variables and arguments Inject credential data into the environment variables and arguments
passed to `ansible-playbook` passed to `ansible-playbook`
@@ -627,9 +627,6 @@ class CredentialType(CommonModelNameNotUnique):
additional arguments based on custom additional arguments based on custom
`extra_vars` injectors defined on this `extra_vars` injectors defined on this
CredentialType. CredentialType.
:param safe_args: a list of arguments stored in the database for
the job run (`UnifiedJob.job_args`); secret
values should be stripped
:param private_data_dir: a temporary directory to store files generated :param private_data_dir: a temporary directory to store files generated
by `file` injectors (like config files or key by `file` injectors (like config files or key
files) files)
@@ -650,7 +647,7 @@ class CredentialType(CommonModelNameNotUnique):
# maintain a normal namespace for building the ansible-playbook arguments (env and args) # maintain a normal namespace for building the ansible-playbook arguments (env and args)
namespace = {'tower': tower_namespace} namespace = {'tower': tower_namespace}
# maintain a sanitized namespace for building the DB-stored arguments (safe_env and safe_args) # maintain a sanitized namespace for building the DB-stored arguments (safe_env)
safe_namespace = {'tower': tower_namespace} safe_namespace = {'tower': tower_namespace}
# build a normal namespace with secret values decrypted (for # build a normal namespace with secret values decrypted (for
@@ -724,7 +721,6 @@ class CredentialType(CommonModelNameNotUnique):
path = build_extra_vars_file(extra_vars, private_data_dir) path = build_extra_vars_file(extra_vars, private_data_dir)
if extra_vars: if extra_vars:
args.extend(['-e', '@%s' % path]) args.extend(['-e', '@%s' % path])
safe_args.extend(['-e', '@%s' % path])
class ManagedCredentialType(SimpleNamespace): class ManagedCredentialType(SimpleNamespace):

191
awx/main/tasks.py
View File

@@ -801,7 +801,7 @@ class BaseTask(object):
'': '', '': '',
} }
def build_extra_vars_file(self, instance, private_data_dir, passwords, display=False): def build_extra_vars_file(self, instance, private_data_dir, passwords):
''' '''
Build ansible yaml file filled with extra vars to be passed via -e@file.yml Build ansible yaml file filled with extra vars to be passed via -e@file.yml
''' '''
@@ -906,9 +906,6 @@ class BaseTask(object):
os.chmod(path, stat.S_IRUSR) os.chmod(path, stat.S_IRUSR)
return path return path
def build_safe_args(self, instance, private_data_dir, passwords):
return self.build_args(instance, private_data_dir, passwords)
def build_cwd(self, instance, private_data_dir): def build_cwd(self, instance, private_data_dir):
raise NotImplementedError raise NotImplementedError
@@ -957,10 +954,10 @@ class BaseTask(object):
''' '''
Run the job/task and capture its output. Run the job/task and capture its output.
''' '''
instance = self.update_model(pk, status='running', self.instance = self.update_model(pk, status='running',
start_args='') # blank field to remove encrypted passwords start_args='') # blank field to remove encrypted passwords
instance.websocket_emit_status("running") self.instance.websocket_emit_status("running")
status, rc, tb = 'error', None, '' status, rc, tb = 'error', None, ''
output_replacements = [] output_replacements = []
extra_update_fields = {} extra_update_fields = {}
@@ -970,75 +967,69 @@ class BaseTask(object):
private_data_dir = None private_data_dir = None
try: try:
isolated = instance.is_isolated() isolated = self.instance.is_isolated()
self.pre_run_hook(instance) self.pre_run_hook(self.instance)
if instance.cancel_flag: if self.instance.cancel_flag:
instance = self.update_model(instance.pk, status='canceled') self.instance = self.update_model(self.instance.pk, status='canceled')
if instance.status != 'running': if self.instance.status != 'running':
# Stop the task chain and prevent starting the job if it has # Stop the task chain and prevent starting the job if it has
# already been canceled. # already been canceled.
instance = self.update_model(pk) self.instance = self.update_model(pk)
status = instance.status status = self.instance.status
raise RuntimeError('not starting %s task' % instance.status) raise RuntimeError('not starting %s task' % self.instance.status)
if not os.path.exists(settings.AWX_PROOT_BASE_PATH): if not os.path.exists(settings.AWX_PROOT_BASE_PATH):
raise RuntimeError('AWX_PROOT_BASE_PATH=%s does not exist' % settings.AWX_PROOT_BASE_PATH) raise RuntimeError('AWX_PROOT_BASE_PATH=%s does not exist' % settings.AWX_PROOT_BASE_PATH)
# store a record of the venv used at runtime # store a record of the venv used at runtime
if hasattr(instance, 'custom_virtualenv'): if hasattr(self.instance, 'custom_virtualenv'):
self.update_model(pk, custom_virtualenv=getattr(instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH)) self.update_model(pk, custom_virtualenv=getattr(self.instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH))
private_data_dir = self.build_private_data_dir(instance) private_data_dir = self.build_private_data_dir(self.instance)
# Fetch "cached" fact data from prior runs and put on the disk # Fetch "cached" fact data from prior runs and put on the disk
# where ansible expects to find it # where ansible expects to find it
if getattr(instance, 'use_fact_cache', False): if getattr(self.instance, 'use_fact_cache', False):
instance.start_job_fact_cache( self.instance.start_job_fact_cache(
os.path.join(private_data_dir, 'artifacts', str(instance.id), 'fact_cache'), os.path.join(private_data_dir, 'artifacts', str(self.instance.id), 'fact_cache'),
fact_modification_times, fact_modification_times,
) )
# May have to serialize the value # May have to serialize the value
private_data_files = self.build_private_data_files(instance, private_data_dir) private_data_files = self.build_private_data_files(self.instance, private_data_dir)
passwords = self.build_passwords(instance, kwargs) passwords = self.build_passwords(self.instance, kwargs)
proot_custom_virtualenv = None proot_custom_virtualenv = None
if getattr(instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH) != settings.ANSIBLE_VENV_PATH: if getattr(self.instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH) != settings.ANSIBLE_VENV_PATH:
proot_custom_virtualenv = instance.ansible_virtualenv_path proot_custom_virtualenv = self.instance.ansible_virtualenv_path
self.build_extra_vars_file(instance, private_data_dir, passwords) self.build_extra_vars_file(self.instance, private_data_dir, passwords)
args = self.build_args(instance, private_data_dir, passwords) args = self.build_args(self.instance, private_data_dir, passwords)
safe_args = self.build_safe_args(instance, private_data_dir, passwords)
# TODO: output_replacements hurts my head right now # TODO: output_replacements hurts my head right now
#output_replacements = self.build_output_replacements(instance, **kwargs) #output_replacements = self.build_output_replacements(self.instance, **kwargs)
output_replacements = [] output_replacements = []
cwd = self.build_cwd(instance, private_data_dir) cwd = self.build_cwd(self.instance, private_data_dir)
env = self.build_env(instance, private_data_dir, isolated, env = self.build_env(self.instance, private_data_dir, isolated,
private_data_files=private_data_files) private_data_files=private_data_files)
safe_env = build_safe_env(env) safe_env = build_safe_env(env)
# handle custom injectors specified on the CredentialType # handle custom injectors specified on the CredentialType
credentials = [] credentials = []
if isinstance(instance, Job): if isinstance(self.instance, Job):
credentials = instance.credentials.all() credentials = self.instance.credentials.all()
elif isinstance(instance, InventoryUpdate): elif isinstance(self.instance, InventoryUpdate):
# TODO: allow multiple custom creds for inv updates # TODO: allow multiple custom creds for inv updates
credentials = [instance.get_cloud_credential()] credentials = [self.instance.get_cloud_credential()]
elif isinstance(instance, Project): elif isinstance(self.instance, Project):
# once (or if) project updates # once (or if) project updates
# move from a .credential -> .credentials model, we can # move from a .credential -> .credentials model, we can
# lose this block # lose this block
credentials = [instance.credential] credentials = [self.instance.credential]
for credential in credentials: for credential in credentials:
if credential: if credential:
credential.credential_type.inject_credential( credential.credential_type.inject_credential(
credential, env, safe_env, args, safe_args, private_data_dir credential, env, safe_env, args, private_data_dir
) )
self.write_args_file(private_data_dir, args) self.write_args_file(private_data_dir, args)
# If we're executing on an isolated host, don't bother adding the
# key to the agent in this environment
instance = self.update_model(pk, job_args=json.dumps(safe_args),
job_cwd=cwd, job_env=safe_env)
expect_passwords = {} expect_passwords = {}
password_prompts = self.get_password_prompts(passwords) password_prompts = self.get_password_prompts(passwords)
for k, v in password_prompts.items(): for k, v in password_prompts.items():
@@ -1047,12 +1038,12 @@ class BaseTask(object):
extra_update_fields=extra_update_fields, extra_update_fields=extra_update_fields,
proot_cmd=getattr(settings, 'AWX_PROOT_CMD', 'bwrap'), proot_cmd=getattr(settings, 'AWX_PROOT_CMD', 'bwrap'),
) )
instance = self.update_model(instance.pk, output_replacements=output_replacements) self.instance = self.update_model(self.instance.pk, output_replacements=output_replacements)
def event_handler(self, instance, event_data): def event_handler(self, event_data):
should_write_event = False should_write_event = False
dispatcher = CallbackQueueDispatcher() dispatcher = CallbackQueueDispatcher()
event_data.setdefault(self.event_data_key, instance.id) event_data.setdefault(self.event_data_key, self.instance.id)
dispatcher.dispatch(event_data) dispatcher.dispatch(event_data)
self.event_ct += 1 self.event_ct += 1
@@ -1060,48 +1051,55 @@ class BaseTask(object):
Handle artifacts Handle artifacts
''' '''
if event_data.get('event_data', {}).get('artifact_data', {}): if event_data.get('event_data', {}).get('artifact_data', {}):
instance.artifacts = event_data['event_data']['artifact_data'] self.instance.artifacts = event_data['event_data']['artifact_data']
instance.save(update_fields=['artifacts']) self.instance.save(update_fields=['artifacts'])
return should_write_event return should_write_event
def cancel_callback(instance): def cancel_callback(self):
instance = self.update_model(pk) self.instance = self.update_model(self.instance.pk)
if instance.cancel_flag or instance.status == 'canceled': if self.instance.cancel_flag or self.instance.status == 'canceled':
cancel_wait = (now() - instance.modified).seconds if instance.modified else 0 cancel_wait = (now() - self.instance.modified).seconds if self.instance.modified else 0
if cancel_wait > 5: if cancel_wait > 5:
logger.warn('Request to cancel {} took {} seconds to complete.'.format(instance.log_format, cancel_wait)) logger.warn('Request to cancel {} took {} seconds to complete.'.format(self.instance.log_format, cancel_wait))
return True return True
return False return False
def finished_callback(self, instance, runner_obj): def finished_callback(self, runner_obj):
dispatcher = CallbackQueueDispatcher() dispatcher = CallbackQueueDispatcher()
event_data = { event_data = {
'event': 'EOF', 'event': 'EOF',
'final_counter': self.event_ct, 'final_counter': self.event_ct,
} }
event_data.setdefault(self.event_data_key, instance.id) event_data.setdefault(self.event_data_key, self.instance.id)
dispatcher.dispatch(event_data) dispatcher.dispatch(event_data)
def status_handler(self, status_data, runner_config):
if status_data['status'] == 'starting':
self.instance = self.update_model(pk, job_args=json.dumps(runner_config.command),
job_cwd=runner_config.cwd, job_env=runner_config.env)
params = { params = {
'ident': instance.id, 'ident': self.instance.id,
'private_data_dir': private_data_dir, 'private_data_dir': private_data_dir,
'project_dir': cwd, 'project_dir': cwd,
'playbook': self.build_playbook_path_relative_to_cwd(instance, private_data_dir), 'playbook': self.build_playbook_path_relative_to_cwd(self.instance, private_data_dir),
'inventory': self.build_inventory(instance, private_data_dir), 'inventory': self.build_inventory(self.instance, private_data_dir),
'passwords': expect_passwords, 'passwords': expect_passwords,
'envvars': env, 'envvars': env,
'event_handler': functools.partial(event_handler, self, instance), 'event_handler': functools.partial(event_handler, self),
'cancel_callback': functools.partial(cancel_callback, instance), 'cancel_callback': functools.partial(cancel_callback, self),
'finished_callback': functools.partial(finished_callback, self, instance), 'finished_callback': functools.partial(finished_callback, self),
'status_handler': functools.partial(status_handler, self),
'settings': { 'settings': {
'idle_timeout': self.get_idle_timeout() or "", 'idle_timeout': self.get_idle_timeout() or "",
'job_timeout': self.get_instance_timeout(instance), 'job_timeout': self.get_instance_timeout(self.instance),
'pexpect_timeout': getattr(settings, 'PEXPECT_TIMEOUT', 5), 'pexpect_timeout': getattr(settings, 'PEXPECT_TIMEOUT', 5),
} }
} }
if self.should_use_proot(instance): if self.should_use_proot(self.instance):
process_isolation_params = { process_isolation_params = {
'process_isolation': True, 'process_isolation': True,
'process_isolation_path': settings.AWX_PROOT_BASE_PATH, 'process_isolation_path': settings.AWX_PROOT_BASE_PATH,
@@ -1126,11 +1124,11 @@ class BaseTask(object):
process_isolation_params['process_isolation_ro_paths'].append(proot_custom_virtualenv) process_isolation_params['process_isolation_ro_paths'].append(proot_custom_virtualenv)
params = {**params, **process_isolation_params} params = {**params, **process_isolation_params}
if isinstance(instance, AdHocCommand): if isinstance(self.instance, AdHocCommand):
params['module'] = self.build_module_name(instance) params['module'] = self.build_module_name(self.instance)
params['module_args'] = self.build_module_args(instance) params['module_args'] = self.build_module_args(self.instance)
if getattr(instance, 'use_fact_cache', False): if getattr(self.instance, 'use_fact_cache', False):
# Enable Ansible fact cache. # Enable Ansible fact cache.
params['fact_cache_type'] = 'jsonfile' params['fact_cache_type'] = 'jsonfile'
else: else:
@@ -1144,7 +1142,7 @@ class BaseTask(object):
if not params[v]: if not params[v]:
del params[v] del params[v]
if instance.is_isolated() is True: if self.instance.is_isolated() is True:
playbook = params['playbook'] playbook = params['playbook']
shutil.move( shutil.move(
params.pop('inventory'), params.pop('inventory'),
@@ -1153,7 +1151,7 @@ class BaseTask(object):
copy_tree(cwd, os.path.join(private_data_dir, 'project')) copy_tree(cwd, os.path.join(private_data_dir, 'project'))
ansible_runner.utils.dump_artifacts(params) ansible_runner.utils.dump_artifacts(params)
manager_instance = isolated_manager.IsolatedManager(env, **_kw) manager_instance = isolated_manager.IsolatedManager(env, **_kw)
status, rc = manager_instance.run(instance, status, rc = manager_instance.run(self.instance,
private_data_dir, private_data_dir,
playbook, playbook,
event_data_key=self.event_data_key) event_data_key=self.event_data_key)
@@ -1163,40 +1161,40 @@ class BaseTask(object):
rc = res.rc rc = res.rc
if status == 'timeout': if status == 'timeout':
instance.job_explanation = "Job terminated due to timeout" self.instance.job_explanation = "Job terminated due to timeout"
status = 'failed' status = 'failed'
extra_update_fields['job_explanation'] = instance.job_explanation extra_update_fields['job_explanation'] = self.instance.job_explanation
except Exception: except Exception:
# run_pexpect does not throw exceptions for cancel or timeout # run_pexpect does not throw exceptions for cancel or timeout
# this could catch programming or file system errors # this could catch programming or file system errors
tb = traceback.format_exc() tb = traceback.format_exc()
logger.exception('%s Exception occurred while running task', instance.log_format) logger.exception('%s Exception occurred while running task', self.instance.log_format)
finally: finally:
logger.info('%s finished running, producing %s events.', instance.log_format, self.event_ct) logger.info('%s finished running, producing %s events.', self.instance.log_format, self.event_ct)
try: try:
self.post_run_hook(instance, status) self.post_run_hook(self.instance, status)
except Exception: except Exception:
logger.exception('{} Post run hook errored.'.format(instance.log_format)) logger.exception('{} Post run hook errored.'.format(self.instance.log_format))
instance = self.update_model(pk) self.instance = self.update_model(pk)
instance = self.update_model(pk, status=status, result_traceback=tb, self.instance = self.update_model(pk, status=status, result_traceback=tb,
output_replacements=output_replacements, output_replacements=output_replacements,
emitted_events=self.event_ct, emitted_events=self.event_ct,
**extra_update_fields) **extra_update_fields)
try: try:
self.final_run_hook(instance, status, private_data_dir, fact_modification_times) self.final_run_hook(self.instance, status, private_data_dir, fact_modification_times)
except Exception: except Exception:
logger.exception('{} Final run hook errored.'.format(instance.log_format)) logger.exception('{} Final run hook errored.'.format(self.instance.log_format))
instance.websocket_emit_status(status) self.instance.websocket_emit_status(status)
if status != 'successful': if status != 'successful':
if status == 'canceled': if status == 'canceled':
raise AwxTaskError.TaskCancel(instance, rc) raise AwxTaskError.TaskCancel(self.instance, rc)
else: else:
raise AwxTaskError.TaskError(instance, rc) raise AwxTaskError.TaskError(self.instance, rc)
@task() @task()
@@ -1357,7 +1355,7 @@ class RunJob(BaseTask):
return env return env
def build_args(self, job, private_data_dir, passwords, display=False): def build_args(self, job, private_data_dir, passwords):
''' '''
Build command line argument list for running ansible-playbook, Build command line argument list for running ansible-playbook,
optionally using ssh-agent for public/private key authentication. optionally using ssh-agent for public/private key authentication.
@@ -1421,9 +1419,6 @@ class RunJob(BaseTask):
return args return args
def build_safe_args(self, job, private_data_dir, passwords):
return self.build_args(job, private_data_dir, passwords, display=True)
def build_cwd(self, job, private_data_dir): def build_cwd(self, job, private_data_dir):
cwd = job.project.get_project_path() cwd = job.project.get_project_path()
if not cwd: if not cwd:
@@ -1435,16 +1430,12 @@ class RunJob(BaseTask):
def build_playbook_path_relative_to_cwd(self, job, private_data_dir): def build_playbook_path_relative_to_cwd(self, job, private_data_dir):
return os.path.join(job.playbook) return os.path.join(job.playbook)
def build_extra_vars_file(self, job, private_data_dir, passwords, display=False): def build_extra_vars_file(self, job, private_data_dir, passwords):
# Define special extra_vars for AWX, combine with job.extra_vars. # Define special extra_vars for AWX, combine with job.extra_vars.
extra_vars = job.awx_meta_vars() extra_vars = job.awx_meta_vars()
if job.extra_vars_dict: if job.extra_vars_dict:
# TODO: Is display needed here? We are building a file that isn't visible extra_vars.update(json.loads(job.decrypted_extra_vars()))
if display and job.job_template:
extra_vars.update(json.loads(job.display_extra_vars()))
else:
extra_vars.update(json.loads(job.decrypted_extra_vars()))
# By default, all extra vars disallow Jinja2 template usage for # By default, all extra vars disallow Jinja2 template usage for
# security reasons; top level key-values defined in JT.extra_vars, however, # security reasons; top level key-values defined in JT.extra_vars, however,
@@ -1688,14 +1679,6 @@ class RunProjectUpdate(BaseTask):
}) })
self._write_extra_vars_file(private_data_dir, extra_vars) self._write_extra_vars_file(private_data_dir, extra_vars)
def build_safe_args(self, project_update, private_data_dir, passwords):
pwdict = dict(passwords.items())
for pw_name, pw_val in list(pwdict.items()):
if pw_name in ('', 'yes', 'no', 'scm_username'):
continue
pwdict[pw_name] = HIDDEN_PASSWORD
return self.build_args(project_update, private_data_dir, passwords)
def build_cwd(self, project_update, private_data_dir): def build_cwd(self, project_update, private_data_dir):
return self.get_path_to('..', 'playbooks') return self.get_path_to('..', 'playbooks')
@@ -2432,7 +2415,7 @@ class RunAdHocCommand(BaseTask):
return args return args
def build_extra_vars_file(self, ad_hoc_command, private_data_dir, passwords={}, display=False): def build_extra_vars_file(self, ad_hoc_command, private_data_dir, passwords={}):
extra_vars = ad_hoc_command.awx_meta_vars() extra_vars = ad_hoc_command.awx_meta_vars()
if ad_hoc_command.extra_vars_dict: if ad_hoc_command.extra_vars_dict:

23
awx/main/tests/unit/models/test_survey_models.py
View File

@@ -132,29 +132,6 @@ def test_survey_passwords_not_in_extra_vars():
} }
def test_job_safe_args_redacted_passwords(job):
"""Verify that safe_args hides passwords in the job extra_vars"""
kwargs = {'ansible_version': '2.1', 'private_data_dir': tempfile.mkdtemp()}
run_job = RunJob()
safe_args = run_job.build_safe_args(job, **kwargs)
ev_index = safe_args.index('-e') + 1
extra_var_file = open(safe_args[ev_index][1:], 'r')
extra_vars = yaml.load(extra_var_file, SafeLoader)
extra_var_file.close()
assert extra_vars['secret_key'] == '$encrypted$'
def test_job_args_unredacted_passwords(job, tmpdir_factory):
kwargs = {'ansible_version': '2.1', 'private_data_dir': tempfile.mkdtemp()}
run_job = RunJob()
args = run_job.build_args(job, **kwargs)
ev_index = args.index('-e') + 1
extra_var_file = open(args[ev_index][1:], 'r')
extra_vars = yaml.load(extra_var_file, SafeLoader)
extra_var_file.close()
assert extra_vars['secret_key'] == 'my_password'
def test_launch_config_has_unprompted_vars(survey_spec_factory): def test_launch_config_has_unprompted_vars(survey_spec_factory):
jt = JobTemplate( jt = JobTemplate(
survey_enabled = True, survey_enabled = True,