From 4c40791d06010f9cbe3313dbcd607a7b010f4144 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 9 Jan 2018 23:32:53 -0500 Subject: [PATCH 01/16] properly handle unicode for isolated job buffers from: https://docs.python.org/2/library/stringio.html#module-cStringIO "Unlike the StringIO module, this module is not able to accept Unicode strings that cannot be encoded as plain ASCII strings." see: https://github.com/ansible/ansible-tower/issues/7846 --- awx/main/expect/isolated_manager.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/awx/main/expect/isolated_manager.py b/awx/main/expect/isolated_manager.py index 265531443f..288717257e 100644 --- a/awx/main/expect/isolated_manager.py +++ b/awx/main/expect/isolated_manager.py @@ -1,5 +1,4 @@ import base64 -import cStringIO import codecs import StringIO import json @@ -143,7 +142,7 @@ class IsolatedManager(object): # if an ssh private key fifo exists, read its contents and delete it if self.ssh_key_path: - buff = cStringIO.StringIO() + buff = StringIO.StringIO() with open(self.ssh_key_path, 'r') as fifo: for line in fifo: buff.write(line) @@ -183,7 +182,7 @@ class IsolatedManager(object): job_timeout=settings.AWX_ISOLATED_LAUNCH_TIMEOUT, pexpect_timeout=5 ) - output = buff.getvalue() + output = buff.getvalue().encode('utf-8') playbook_logger.info('Isolated job {} dispatch:\n{}'.format(self.instance.id, output)) if status != 'successful': self.stdout_handle.write(output) @@ -283,7 +282,7 @@ class IsolatedManager(object): status = 'failed' output = '' rc = None - buff = cStringIO.StringIO() + buff = StringIO.StringIO() last_check = time.time() seek = 0 job_timeout = remaining = self.job_timeout @@ -304,7 +303,7 @@ class IsolatedManager(object): time.sleep(1) continue - buff = cStringIO.StringIO() + buff = StringIO.StringIO() logger.debug('Checking on isolated job {} with `check_isolated.yml`.'.format(self.instance.id)) status, rc = IsolatedManager.run_pexpect( args, self.awx_playbook_path(), self.management_env, buff, @@ -314,7 +313,7 @@ class IsolatedManager(object): pexpect_timeout=5, proot_cmd=self.proot_cmd ) - output = buff.getvalue() + output = buff.getvalue().encode('utf-8') playbook_logger.info('Isolated job {} check:\n{}'.format(self.instance.id, output)) path = self.path_to('artifacts', 'stdout') @@ -356,14 +355,14 @@ class IsolatedManager(object): } args = self._build_args('clean_isolated.yml', '%s,' % self.host, extra_vars) logger.debug('Cleaning up job {} on isolated host with `clean_isolated.yml` playbook.'.format(self.instance.id)) - buff = cStringIO.StringIO() + buff = StringIO.StringIO() timeout = max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT) status, rc = IsolatedManager.run_pexpect( args, self.awx_playbook_path(), self.management_env, buff, idle_timeout=timeout, job_timeout=timeout, pexpect_timeout=5 ) - output = buff.getvalue() + output = buff.getvalue().encode('utf-8') playbook_logger.info('Isolated job {} cleanup:\n{}'.format(self.instance.id, output)) if status != 'successful': @@ -406,14 +405,14 @@ class IsolatedManager(object): env = cls._base_management_env() env['ANSIBLE_STDOUT_CALLBACK'] = 'json' - buff = cStringIO.StringIO() + buff = StringIO.StringIO() timeout = max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT) status, rc = IsolatedManager.run_pexpect( args, cls.awx_playbook_path(), env, buff, idle_timeout=timeout, job_timeout=timeout, pexpect_timeout=5 ) - output = buff.getvalue() + output = buff.getvalue().encode('utf-8') buff.close() try: From b8758044e0a89d6543cc1fe46e4b11be48e51a10 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 10 Jan 2018 19:10:57 -0500 Subject: [PATCH 02/16] fix a bug in inventory generation for isolated nodes see: https://github.com/ansible/ansible-tower/issues/7849 related: https://github.com/ansible/awx/pull/551 --- awx/main/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 822e5b89f9..e2528d197d 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -707,7 +707,7 @@ class BaseTask(LogErrorsTask): 'hostvars': True, }).run(buff) json_data = buff.getvalue().strip() - f.write("#! /usr/bin/env python\nprint '''%s'''\n" % json_data) + f.write('#! /usr/bin/env python\n# -*- coding: utf-8 -*-\nprint %r\n' % json_data) os.chmod(path, stat.S_IRUSR | stat.S_IXUSR) return path else: From 563f73026833aa6b831b251777d8e6c1b3ac5e98 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 12 Jan 2018 10:40:40 -0500 Subject: [PATCH 03/16] add support for new "BECOME" prompt in Ansible 2.5+ see: https://github.com/ansible/ansible-tower/issues/7850 --- awx/main/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index e2528d197d..71552d329b 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1217,6 +1217,7 @@ class RunJob(BaseTask): for method in PRIVILEGE_ESCALATION_METHODS: d[re.compile(r'%s password.*:\s*?$' % (method[0]), re.M)] = 'become_password' d[re.compile(r'%s password.*:\s*?$' % (method[0].upper()), re.M)] = 'become_password' + d[re.compile(r'BECOME password.*:\s*?$', re.M)] = 'become_password' d[re.compile(r'SSH password:\s*?$', re.M)] = 'ssh_password' d[re.compile(r'Password:\s*?$', re.M)] = 'ssh_password' d[re.compile(r'Vault password:\s*?$', re.M)] = 'vault_password' From 983b192a4541004fa09a1f266129390dc78c45e2 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 12 Jan 2018 17:27:54 -0500 Subject: [PATCH 04/16] replace our memcached-based fact cache implementation with local files see: https://github.com/ansible/ansible-tower/issues/7840 --- awx/main/models/jobs.py | 132 +++++++---------- awx/main/tasks.py | 23 ++- awx/main/tests/unit/models/test_jobs.py | 189 ++++++++++-------------- awx/main/tests/unit/test_tasks.py | 17 +++ awx/plugins/fact_caching/awx.py | 128 ---------------- docs/fact_cache.md | 45 ++++-- 6 files changed, 198 insertions(+), 336 deletions(-) delete mode 100755 awx/plugins/fact_caching/awx.py diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index d0729cc48b..4f5e549e04 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -2,26 +2,26 @@ # All Rights Reserved. # Python +import codecs import datetime import hashlib import hmac import logging +import os import time import json -import base64 from urlparse import urljoin +import six + # Django from django.conf import settings from django.db import models #from django.core.cache import cache -import memcache from django.db.models import Q, Count from django.utils.dateparse import parse_datetime -from dateutil import parser -from dateutil.tz import tzutc from django.utils.encoding import force_text, smart_str -from django.utils.timezone import utc +from django.utils.timezone import utc, now from django.utils.translation import ugettext_lazy as _ from django.core.exceptions import ValidationError @@ -714,85 +714,61 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana def get_notification_friendly_name(self): return "Job" - @property - def memcached_fact_key(self): - return '{}'.format(self.inventory.id) - - def memcached_fact_host_key(self, host_name): - return '{}-{}'.format(self.inventory.id, base64.b64encode(host_name.encode('utf-8'))) - - def memcached_fact_modified_key(self, host_name): - return '{}-{}-modified'.format(self.inventory.id, base64.b64encode(host_name.encode('utf-8'))) - - def _get_inventory_hosts(self, only=['name', 'ansible_facts', 'modified',]): + def _get_inventory_hosts(self, only=['name', 'ansible_facts', 'ansible_facts_modified', 'modified',]): + if not self.inventory: + return [] return self.inventory.hosts.only(*only) - def _get_memcache_connection(self): - return memcache.Client([settings.CACHES['default']['LOCATION']], debug=0) - - def start_job_fact_cache(self): - if not self.inventory: - return - - cache = self._get_memcache_connection() - - host_names = [] - - for host in self._get_inventory_hosts(): - host_key = self.memcached_fact_host_key(host.name) - modified_key = self.memcached_fact_modified_key(host.name) - - if cache.get(modified_key) is None: - if host.ansible_facts_modified: - host_modified = host.ansible_facts_modified.replace(tzinfo=tzutc()).isoformat() - else: - host_modified = datetime.datetime.now(tzutc()).isoformat() - cache.set(host_key, json.dumps(host.ansible_facts)) - cache.set(modified_key, host_modified) - - host_names.append(host.name) - - cache.set(self.memcached_fact_key, host_names) - - def finish_job_fact_cache(self): - if not self.inventory: - return - - cache = self._get_memcache_connection() - + def start_job_fact_cache(self, destination, modification_times, timeout=None): + destination = os.path.join(destination, 'facts') + os.makedirs(destination, mode=0700) hosts = self._get_inventory_hosts() + if timeout is None: + timeout = settings.ANSIBLE_FACT_CACHE_TIMEOUT + if timeout > 0: + # exclude hosts with fact data older than `settings.ANSIBLE_FACT_CACHE_TIMEOUT seconds` + timeout = now() - datetime.timedelta(seconds=timeout) + hosts = hosts.filter(ansible_facts_modified__gte=timeout) for host in hosts: - host_key = self.memcached_fact_host_key(host.name) - modified_key = self.memcached_fact_modified_key(host.name) + filepath = os.sep.join(map(six.text_type, [destination, host.name])) + with codecs.open(filepath, 'w', encoding='utf-8') as f: + os.chmod(f.name, 0600) + json.dump(host.ansible_facts, f) + # make note of the time we wrote the file so we can check if it changed later + modification_times[filepath] = os.path.getmtime(filepath) - modified = cache.get(modified_key) - if modified is None: - cache.delete(host_key) - continue - - # Save facts if cache is newer than DB - modified = parser.parse(modified, tzinfos=[tzutc()]) - if not host.ansible_facts_modified or modified > host.ansible_facts_modified: - ansible_facts = cache.get(host_key) - try: - ansible_facts = json.loads(ansible_facts) - except Exception: - ansible_facts = None - - if ansible_facts is None: - cache.delete(host_key) - continue - host.ansible_facts = ansible_facts - host.ansible_facts_modified = modified - if 'insights' in ansible_facts and 'system_id' in ansible_facts['insights']: - host.insights_system_id = ansible_facts['insights']['system_id'] - host.save() + def finish_job_fact_cache(self, destination, modification_times): + destination = os.path.join(destination, 'facts') + for host in self._get_inventory_hosts(): + filepath = os.sep.join(map(six.text_type, [destination, host.name])) + if os.path.exists(filepath): + # If the file changed since we wrote it pre-playbook run... + modified = os.path.getmtime(filepath) + if modified > modification_times.get(filepath, 0): + with codecs.open(filepath, 'r', encoding='utf-8') as f: + try: + ansible_facts = json.load(f) + except ValueError: + continue + host.ansible_facts = ansible_facts + host.ansible_facts_modified = now() + if 'insights' in ansible_facts and 'system_id' in ansible_facts['insights']: + host.insights_system_id = ansible_facts['insights']['system_id'] + host.save() + system_tracking_logger.info( + 'New fact for inventory {} host {}'.format( + smart_str(host.inventory.name), smart_str(host.name)), + extra=dict(inventory_id=host.inventory.id, host_name=host.name, + ansible_facts=host.ansible_facts, + ansible_facts_modified=host.ansible_facts_modified.isoformat())) + else: + # if the file goes missing, ansible removed it (likely via clear_facts) + host.ansible_facts = {} + host.ansible_facts_modified = now() system_tracking_logger.info( - 'New fact for inventory {} host {}'.format( - smart_str(host.inventory.name), smart_str(host.name)), - extra=dict(inventory_id=host.inventory.id, host_name=host.name, - ansible_facts=host.ansible_facts, - ansible_facts_modified=host.ansible_facts_modified.isoformat())) + 'Facts cleared for inventory {} host {}'.format( + smart_str(host.inventory.name), smart_str(host.name))) + host.save() class JobHostSummary(CreatedModifiedModel): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 71552d329b..b60ee6b16d 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -812,6 +812,15 @@ class BaseTask(LogErrorsTask): # Fetch ansible version once here to support version-dependent features. kwargs['ansible_version'] = get_ansible_version() kwargs['private_data_dir'] = self.build_private_data_dir(instance, **kwargs) + + # Fetch "cached" fact data from prior runs and put on the disk + # where ansible expects to find it + if getattr(instance, 'use_fact_cache', False) and not kwargs.get('isolated'): + instance.start_job_fact_cache( + os.path.join(kwargs['private_data_dir']), + kwargs.setdefault('fact_modification_times', {}) + ) + # May have to serialize the value kwargs['private_data_files'] = self.build_private_data_files(instance, **kwargs) kwargs['passwords'] = self.build_passwords(instance, **kwargs) @@ -1032,10 +1041,8 @@ class RunJob(BaseTask): env['INVENTORY_ID'] = str(job.inventory.pk) if job.use_fact_cache and not kwargs.get('isolated'): env['ANSIBLE_LIBRARY'] = self.get_path_to('..', 'plugins', 'library') - env['ANSIBLE_CACHE_PLUGINS'] = self.get_path_to('..', 'plugins', 'fact_caching') - env['ANSIBLE_CACHE_PLUGIN'] = "awx" - env['ANSIBLE_CACHE_PLUGIN_TIMEOUT'] = str(settings.ANSIBLE_FACT_CACHE_TIMEOUT) - env['ANSIBLE_CACHE_PLUGIN_CONNECTION'] = settings.CACHES['default']['LOCATION'] if 'LOCATION' in settings.CACHES['default'] else '' + env['ANSIBLE_CACHE_PLUGIN'] = "jsonfile" + env['ANSIBLE_CACHE_PLUGIN_CONNECTION'] = os.path.join(kwargs['private_data_dir'], 'facts') if job.project: env['PROJECT_REVISION'] = job.project.scm_revision env['ANSIBLE_RETRY_FILES_ENABLED'] = "False" @@ -1286,14 +1293,14 @@ class RunJob(BaseTask): ('project_update', local_project_sync.name, local_project_sync.id))) raise - if job.use_fact_cache and not kwargs.get('isolated'): - job.start_job_fact_cache() - def final_run_hook(self, job, status, **kwargs): super(RunJob, self).final_run_hook(job, status, **kwargs) if job.use_fact_cache and not kwargs.get('isolated'): - job.finish_job_fact_cache() + job.finish_job_fact_cache( + kwargs['private_data_dir'], + kwargs['fact_modification_times'] + ) try: inventory = job.inventory except Inventory.DoesNotExist: diff --git a/awx/main/tests/unit/models/test_jobs.py b/awx/main/tests/unit/models/test_jobs.py index e60b775066..f8949a0333 100644 --- a/awx/main/tests/unit/models/test_jobs.py +++ b/awx/main/tests/unit/models/test_jobs.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +import json +import os +import time import pytest @@ -8,51 +11,14 @@ from awx.main.models import ( Host, ) -import datetime -import json -import base64 -from dateutil.tz import tzutc - - -class CacheMock(object): - def __init__(self): - self.d = dict() - - def get(self, key): - if key not in self.d: - return None - return self.d[key] - - def set(self, key, val): - self.d[key] = val - - def delete(self, key): - del self.d[key] - @pytest.fixture -def old_time(): - return (datetime.datetime.now(tzutc()) - datetime.timedelta(minutes=60)) - - -@pytest.fixture() -def new_time(): - return (datetime.datetime.now(tzutc())) - - -@pytest.fixture -def hosts(old_time, inventory): +def hosts(inventory): return [ - Host(name='host1', ansible_facts={"a": 1, "b": 2}, ansible_facts_modified=old_time, inventory=inventory), - Host(name='host2', ansible_facts={"a": 1, "b": 2}, ansible_facts_modified=old_time, inventory=inventory), - Host(name='host3', ansible_facts={"a": 1, "b": 2}, ansible_facts_modified=old_time, inventory=inventory), - ] - - -@pytest.fixture -def hosts2(inventory): - return [ - Host(name='host2', ansible_facts="foobar", ansible_facts_modified=old_time, inventory=inventory), + Host(name='host1', ansible_facts={"a": 1, "b": 2}, inventory=inventory), + Host(name='host2', ansible_facts={"a": 1, "b": 2}, inventory=inventory), + Host(name='host3', ansible_facts={"a": 1, "b": 2}, inventory=inventory), + Host(name=u'Iñtërnâtiônàlizætiøn', ansible_facts={"a": 1, "b": 2}, inventory=inventory), ] @@ -62,87 +28,92 @@ def inventory(): @pytest.fixture -def mock_cache(mocker): - cache = CacheMock() - mocker.patch.object(cache, 'set', wraps=cache.set) - mocker.patch.object(cache, 'get', wraps=cache.get) - mocker.patch.object(cache, 'delete', wraps=cache.delete) - return cache - - -@pytest.fixture -def job(mocker, hosts, inventory, mock_cache): +def job(mocker, hosts, inventory): j = Job(inventory=inventory, id=2) j._get_inventory_hosts = mocker.Mock(return_value=hosts) - j._get_memcache_connection = mocker.Mock(return_value=mock_cache) return j -@pytest.fixture -def job2(mocker, hosts2, inventory, mock_cache): - j = Job(inventory=inventory, id=3) - j._get_inventory_hosts = mocker.Mock(return_value=hosts2) - j._get_memcache_connection = mocker.Mock(return_value=mock_cache) - return j +def test_start_job_fact_cache(hosts, job, inventory, tmpdir): + fact_cache = str(tmpdir) + modified_times = {} + job.start_job_fact_cache(fact_cache, modified_times, 0) + + for host in hosts: + filepath = os.path.join(fact_cache, 'facts', host.name) + assert os.path.exists(filepath) + with open(filepath, 'r') as f: + assert f.read() == json.dumps(host.ansible_facts) + assert filepath in modified_times -def test_start_job_fact_cache(hosts, job, inventory, mocker): +def test_finish_job_fact_cache_with_existing_data(job, hosts, inventory, mocker, tmpdir): + fact_cache = str(tmpdir) + modified_times = {} + job.start_job_fact_cache(fact_cache, modified_times, 0) - job.start_job_fact_cache() - - job._get_memcache_connection().set.assert_any_call('5', [h.name for h in hosts]) - for host in hosts: - job._get_memcache_connection().set.assert_any_call('{}-{}'.format(5, base64.b64encode(host.name)), json.dumps(host.ansible_facts)) - job._get_memcache_connection().set.assert_any_call('{}-{}-modified'.format(5, base64.b64encode(host.name)), host.ansible_facts_modified.isoformat()) - - -def test_start_job_fact_cache_existing_host(hosts, hosts2, job, job2, inventory, mocker): - - job.start_job_fact_cache() - - for host in hosts: - job._get_memcache_connection().set.assert_any_call('{}-{}'.format(5, base64.b64encode(host.name)), json.dumps(host.ansible_facts)) - job._get_memcache_connection().set.assert_any_call('{}-{}-modified'.format(5, base64.b64encode(host.name)), host.ansible_facts_modified.isoformat()) - - job._get_memcache_connection().set.reset_mock() - - job2.start_job_fact_cache() - - # Ensure hosts2 ansible_facts didn't overwrite hosts ansible_facts - ansible_facts_cached = job._get_memcache_connection().get('{}-{}'.format(5, base64.b64encode(hosts2[0].name))) - assert ansible_facts_cached == json.dumps(hosts[1].ansible_facts) - - -def test_memcached_fact_host_key_unicode(job): - host_name = u'Iñtërnâtiônàlizætiøn' - host_key = job.memcached_fact_host_key(host_name) - assert host_key == '5-ScOxdMOrcm7DonRpw7Ruw6BsaXrDpnRpw7hu' - - -def test_memcached_fact_modified_key_unicode(job): - host_name = u'Iñtërnâtiônàlizætiøn' - host_key = job.memcached_fact_modified_key(host_name) - assert host_key == '5-ScOxdMOrcm7DonRpw7Ruw6BsaXrDpnRpw7hu-modified' - - -def test_finish_job_fact_cache(job, hosts, inventory, mocker, new_time): - - job.start_job_fact_cache() for h in hosts: h.save = mocker.Mock() - host_key = job.memcached_fact_host_key(hosts[1].name) - modified_key = job.memcached_fact_modified_key(hosts[1].name) - ansible_facts_new = {"foo": "bar", "insights": {"system_id": "updated_by_scan"}} - job._get_memcache_connection().set(host_key, json.dumps(ansible_facts_new)) - job._get_memcache_connection().set(modified_key, new_time.isoformat()) - - job.finish_job_fact_cache() + filepath = os.path.join(fact_cache, 'facts', hosts[1].name) + with open(filepath, 'w') as f: + f.write(json.dumps(ansible_facts_new)) + f.flush() + # I feel kind of gross about calling `os.utime` by hand, but I noticed + # that in our container-based dev environment, the resolution for + # `os.stat()` after a file write was over a second, and I don't want to put + # a sleep() in this test + new_modification_time = time.time() + 3600 + os.utime(filepath, (new_modification_time, new_modification_time)) - hosts[0].save.assert_not_called() - hosts[2].save.assert_not_called() + job.finish_job_fact_cache(fact_cache, modified_times) + + for host in (hosts[0], hosts[2], hosts[3]): + host.save.assert_not_called() + assert host.ansible_facts == {"a": 1, "b": 2} + assert host.ansible_facts_modified is None assert hosts[1].ansible_facts == ansible_facts_new assert hosts[1].insights_system_id == "updated_by_scan" hosts[1].save.assert_called_once_with() + +def test_finish_job_fact_cache_with_bad_data(job, hosts, inventory, mocker, tmpdir): + fact_cache = str(tmpdir) + modified_times = {} + job.start_job_fact_cache(fact_cache, modified_times, 0) + + for h in hosts: + h.save = mocker.Mock() + + for h in hosts: + filepath = os.path.join(fact_cache, 'facts', h.name) + with open(filepath, 'w') as f: + f.write('not valid json!') + f.flush() + new_modification_time = time.time() + 3600 + os.utime(filepath, (new_modification_time, new_modification_time)) + + job.finish_job_fact_cache(fact_cache, modified_times) + + for h in hosts: + h.save.assert_not_called() + + +def test_finish_job_fact_cache_clear(job, hosts, inventory, mocker, tmpdir): + fact_cache = str(tmpdir) + modified_times = {} + job.start_job_fact_cache(fact_cache, modified_times, 0) + + for h in hosts: + h.save = mocker.Mock() + + os.remove(os.path.join(fact_cache, 'facts', hosts[1].name)) + job.finish_job_fact_cache(fact_cache, modified_times) + + for host in (hosts[0], hosts[2], hosts[3]): + host.save.assert_not_called() + assert host.ansible_facts == {"a": 1, "b": 2} + assert host.ansible_facts_modified is None + assert hosts[1].ansible_facts == {} + hosts[1].save.assert_called_once_with() diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index f5a8c309b6..963ebc87ae 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -331,6 +331,23 @@ class TestGenericRun(TestJobExecution): args, cwd, env, stdout = call_args assert env['FOO'] == 'BAR' + def test_fact_cache_usage(self): + self.instance.use_fact_cache = True + + start_mock = mock.Mock() + patch = mock.patch.object(Job, 'start_job_fact_cache', start_mock) + self.patches.append(patch) + patch.start() + + self.task.run(self.pk) + call_args, _ = self.run_pexpect.call_args_list[0] + args, cwd, env, stdout = call_args + start_mock.assert_called_once() + tmpdir, _ = start_mock.call_args[0] + + assert env['ANSIBLE_CACHE_PLUGIN'] == 'jsonfile' + assert env['ANSIBLE_CACHE_PLUGIN_CONNECTION'] == os.path.join(tmpdir, 'facts') + class TestAdhocRun(TestJobExecution): diff --git a/awx/plugins/fact_caching/awx.py b/awx/plugins/fact_caching/awx.py deleted file mode 100755 index 9b929a2728..0000000000 --- a/awx/plugins/fact_caching/awx.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (c) 2015 Ansible, Inc. -# This file is a utility Ansible plugin that is not part of the AWX or Ansible -# packages. It does not import any code from either package, nor does its -# license apply to Ansible or AWX. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# Neither the name of the nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -import os -import memcache -import json -import datetime -import base64 -from dateutil import parser -from dateutil.tz import tzutc - -from ansible import constants as C - -try: - from ansible.cache.base import BaseCacheModule -except Exception: - from ansible.plugins.cache.base import BaseCacheModule - - -class CacheModule(BaseCacheModule): - - def __init__(self, *args, **kwargs): - self.mc = memcache.Client([C.CACHE_PLUGIN_CONNECTION], debug=0) - self._timeout = int(C.CACHE_PLUGIN_TIMEOUT) - self._inventory_id = os.environ['INVENTORY_ID'] - - @property - def host_names_key(self): - return '{}'.format(self._inventory_id) - - def translate_host_key(self, host_name): - return '{}-{}'.format(self._inventory_id, base64.b64encode(host_name.encode('utf-8'))) - - def translate_modified_key(self, host_name): - return '{}-{}-modified'.format(self._inventory_id, base64.b64encode(host_name.encode('utf-8'))) - - def get(self, key): - host_key = self.translate_host_key(key) - modified_key = self.translate_modified_key(key) - - ''' - Cache entry expired - ''' - modified = self.mc.get(modified_key) - if modified is None: - raise KeyError - modified = parser.parse(modified).replace(tzinfo=tzutc()) - now_utc = datetime.datetime.now(tzutc()) - if self._timeout != 0 and (modified + datetime.timedelta(seconds=self._timeout)) < now_utc: - raise KeyError - - value_json = self.mc.get(host_key) - if value_json is None: - raise KeyError - try: - return json.loads(value_json) - # If cache entry is corrupt or bad, fail gracefully. - except (TypeError, ValueError): - self.delete(key) - raise KeyError - - def set(self, key, value): - host_key = self.translate_host_key(key) - modified_key = self.translate_modified_key(key) - - self.mc.set(host_key, json.dumps(value)) - self.mc.set(modified_key, datetime.datetime.now(tzutc()).isoformat()) - - def keys(self): - return self.mc.get(self.host_names_key) - - def contains(self, key): - try: - self.get(key) - return True - except KeyError: - return False - - def delete(self, key): - self.set(key, {}) - - def flush(self): - host_names = self.mc.get(self.host_names_key) - if not host_names: - return - - for k in host_names: - self.mc.delete(self.translate_host_key(k)) - self.mc.delete(self.translate_modified_key(k)) - - def copy(self): - ret = dict() - host_names = self.mc.get(self.host_names_key) - if not host_names: - return - - for k in host_names: - ret[k] = self.mc.get(self.translate_host_key(k)) - return ret - diff --git a/docs/fact_cache.md b/docs/fact_cache.md index d1716d8aec..72ad41b759 100644 --- a/docs/fact_cache.md +++ b/docs/fact_cache.md @@ -1,21 +1,40 @@ -# Tower as an Ansible Fact Cache -Tower can store and retrieve per-host facts via an Ansible Fact Cache Plugin. This behavior is configurable on a per-job-template basis. When enabled, Tower will serve fact requests for all Hosts in an Inventory related to the Job running. This allows users to use Job Templates with `--limit` while still having access to the entire Inventory of Host facts. The Tower Ansible Fact Cache supports a global timeout settings that it enforces per-host. The setting is available in the CTiT interface under the Jobs category with the name `ANSIBLE_FACT_CACHE_TIMEOUT` and is in seconds. +# AWX as an Ansible Fact Cache +AWX can store and retrieve per-host facts via an Ansible Fact Cache Plugin. +This behavior is configurable on a per-job-template basis. When enabled, AWX +will serve fact requests for all Hosts in an Inventory related to the Job +running. This allows users to use Job Templates with `--limit` while still +having access to the entire Inventory of Host facts. -## Tower Fact Cache Implementation Details -### Tower Injection -In order to understand the behavior of Tower as a fact cache you will need to understand how fact caching is achieved in Tower. Upon a Job invocation with `use_fact_cache=True`, Tower will inject, into memcached, all `ansible_facts` associated with each Host in the Inventory associated with the Job. Jobs invoked with `use_fact_cache=False` will not inject `ansible_facts` into memcached. The cache key is of the form `inventory_id-host_name` so that hosts with the same name in different inventories do not clash. A list of all hosts in the inventory is also injected into memcached with key `inventory_id` and value `[host_name1, host_name2, ..., host_name3]`. +## AWX Fact Cache Implementation Details +### AWX Injection +In order to understand the behavior of AWX as a fact cache you will need to +understand how fact caching is achieved in AWX. When a Job launches with +`use_fact_cache=True`, AWX will write all `ansible_facts` associated with +each Host in the associated Inventory as JSON files on the local file system +(one JSON file per host). Jobs invoked with `use_fact_cache=False` will not +write `ansible_facts` files. ### Ansible plugin usage -The Ansible fact cache plugin that ships with Tower will only be enabled on Jobs that have fact cache enabled, `use_fact_cache=True`. The fact cache plugin running in Ansible will connect to the same memcached instance. A `get()` call to the fact cache interface in Ansible will result in a looked into memcached for the host-specific set of facts. A `set()` call to the fact cache will result in an update to memcached record along with the modified time. +When `use_fact_cache=True`, Ansible will be configured to use the `jsonfile` +cache plugin. Any `get()` call to the fact cache interface in Ansible will +result in a JSON file lookup for the host-specific set of facts. Any `set()` +call to the fact cache will result in a JSON file being written to the local +file system. -### Tower Cache to DB -When a Job finishes running that has `use_fact_cache=True` enabled, Tower will go through memcached and get all records for the hosts in the Inventory. Any records with update times newer than the database per-host `ansible_facts_modified` value will result in the `ansible_facts`, `ansible_facts_modified` from memcached being saved to the database. Note that the last value of the Ansible fact cache is retained in `ansible_facts`. The globla timeout and/or individual job template `use_fact_cache` setting will not clear the per-host `ansible_facts`. +### AWX Cache to DB +When a Job with `use_fact_cache=True` finishes running, AWX will look at all +of the local JSON files that represent the fact data. Any records with file +modification times that have increased (because Ansible updated the file via +`cache.set()`) will result in the latest value being saved to the database. On +subsequent playbook runs, AWX will _only_ inject cached facts that are _newer_ +than `settings.ANSIBLE_FACT_CACHE_TIMEOUT` seconds. -### Caching Behavior -Tower will always inject the host `ansible_facts` into memcached. The Ansible Tower Fact Cache Plugin will choose to present the cached values to the user or not based on the per-host `ansible_facts_modified` time and the global `ANSIBLE_FACT_CACHE_TIMEOUT`. - -## Tower Fact Logging -New and changed facts will be logged via Tower's logging facility. Specifically, to the `system_tracking` namespace or logger. The logging payload will include the fields: `host_name`, `inventory_id`, and `ansible_facts`. Where `ansible_facts` is a dictionary of all ansible facts for `host_name` in Tower Inventory `inventory_id`. +## AWX Fact Logging +New and changed facts will be logged via AWX's logging facility. Specifically, +to the `system_tracking` namespace or logger. The logging payload will include +the fields: `host_name`, `inventory_id`, and `ansible_facts`. Where +`ansible_facts` is a dictionary of all ansible facts for `host_name` in AWX +Inventory `inventory_id`. ## Integration Testing * ensure `clear_facts` set's `hosts//ansible_facts` to `{}` From e1d50a43fd90983bbb385b5d7ad8aa8a9c63b29a Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 15 Jan 2018 11:38:02 -0500 Subject: [PATCH 05/16] only allow facts to cache in the proper file system location --- awx/main/models/jobs.py | 6 ++++++ awx/main/tests/unit/models/test_jobs.py | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 4f5e549e04..07afd288ce 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -731,6 +731,9 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana hosts = hosts.filter(ansible_facts_modified__gte=timeout) for host in hosts: filepath = os.sep.join(map(six.text_type, [destination, host.name])) + if not os.path.realpath(filepath).startswith(destination): + system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name))) + continue with codecs.open(filepath, 'w', encoding='utf-8') as f: os.chmod(f.name, 0600) json.dump(host.ansible_facts, f) @@ -741,6 +744,9 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana destination = os.path.join(destination, 'facts') for host in self._get_inventory_hosts(): filepath = os.sep.join(map(six.text_type, [destination, host.name])) + if not os.path.realpath(filepath).startswith(destination): + system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name))) + continue if os.path.exists(filepath): # If the file changed since we wrote it pre-playbook run... modified = os.path.getmtime(filepath) diff --git a/awx/main/tests/unit/models/test_jobs.py b/awx/main/tests/unit/models/test_jobs.py index f8949a0333..516a6f076f 100644 --- a/awx/main/tests/unit/models/test_jobs.py +++ b/awx/main/tests/unit/models/test_jobs.py @@ -47,6 +47,17 @@ def test_start_job_fact_cache(hosts, job, inventory, tmpdir): assert filepath in modified_times +def test_fact_cache_with_invalid_path_traversal(job, inventory, tmpdir, mocker): + job._get_inventory_hosts = mocker.Mock(return_value=[ + Host(name='../foo', ansible_facts={"a": 1, "b": 2},), + ]) + + fact_cache = str(tmpdir) + job.start_job_fact_cache(fact_cache, {}, 0) + # a file called "foo" should _not_ be written outside the facts dir + assert os.listdir(os.path.join(fact_cache, 'facts', '..')) == ['facts'] + + def test_finish_job_fact_cache_with_existing_data(job, hosts, inventory, mocker, tmpdir): fact_cache = str(tmpdir) modified_times = {} From 2955842c44c69cc4b93811ac8311abdf48ba28bc Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 15 Jan 2018 13:35:51 -0500 Subject: [PATCH 06/16] don't overwrite env['ANSIBLE_LIBRARY'] when fact caching is enabled see: https://github.com/ansible/awx/issues/815 see: https://github.com/ansible/ansible-tower/issues/7830 --- awx/main/tasks.py | 8 +++++++- awx/main/tests/unit/test_tasks.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index b60ee6b16d..3345dada44 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1040,7 +1040,13 @@ class RunJob(BaseTask): env['JOB_ID'] = str(job.pk) env['INVENTORY_ID'] = str(job.inventory.pk) if job.use_fact_cache and not kwargs.get('isolated'): - env['ANSIBLE_LIBRARY'] = self.get_path_to('..', 'plugins', 'library') + library_path = env.get('ANSIBLE_LIBRARY') + env['ANSIBLE_LIBRARY'] = ':'.join( + filter(None, [ + library_path, + self.get_path_to('..', 'plugins', 'library') + ]) + ) env['ANSIBLE_CACHE_PLUGIN'] = "jsonfile" env['ANSIBLE_CACHE_PLUGIN_CONNECTION'] = os.path.join(kwargs['private_data_dir'], 'facts') if job.project: diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 963ebc87ae..20a051ec30 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -348,6 +348,25 @@ class TestGenericRun(TestJobExecution): assert env['ANSIBLE_CACHE_PLUGIN'] == 'jsonfile' assert env['ANSIBLE_CACHE_PLUGIN_CONNECTION'] == os.path.join(tmpdir, 'facts') + @pytest.mark.parametrize('task_env, ansible_library_env', [ + [{}, '/awx_devel/awx/plugins/library'], + [{'ANSIBLE_LIBRARY': '/foo/bar'}, '/foo/bar:/awx_devel/awx/plugins/library'], + ]) + def test_fact_cache_usage_with_ansible_library(self, task_env, ansible_library_env): + patch = mock.patch('awx.main.tasks.settings.AWX_TASK_ENV', task_env) + patch.start() + + self.instance.use_fact_cache = True + start_mock = mock.Mock() + patch = mock.patch.object(Job, 'start_job_fact_cache', start_mock) + self.patches.append(patch) + patch.start() + + self.task.run(self.pk) + call_args, _ = self.run_pexpect.call_args_list[0] + args, cwd, env, stdout = call_args + assert env['ANSIBLE_LIBRARY'] == ansible_library_env + class TestAdhocRun(TestJobExecution): From d57470ce49f80afd6b510667be151b035f7b0e55 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 23 Jan 2018 15:42:51 -0500 Subject: [PATCH 07/16] don't process artifacts from custom `set_stat` calls asynchronously previously, we persisted custom artifacts to the database on `Job.artifacts` via the callback receiver. when the callback receiver is backed up processing events, this can result in race conditions for workflows where a playbook calls `set_stat()`, but the artifact data is not persisted in the database before the next job in the workflow starts see: https://github.com/ansible/ansible-tower/issues/7831 --- awx/lib/awx_display_callback/module.py | 20 ++++++++++++++++++-- awx/lib/tests/test_display_callback.py | 25 +++++++++++++++++++++++++ awx/main/tasks.py | 16 ++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/awx/lib/awx_display_callback/module.py b/awx/lib/awx_display_callback/module.py index 6800560cfc..368063d0d1 100644 --- a/awx/lib/awx_display_callback/module.py +++ b/awx/lib/awx_display_callback/module.py @@ -18,7 +18,11 @@ from __future__ import (absolute_import, division, print_function) # Python +import codecs import contextlib +import json +import os +import stat import sys import uuid from copy import copy @@ -292,10 +296,22 @@ class BaseCallbackModule(CallbackBase): failures=stats.failures, ok=stats.ok, processed=stats.processed, - skipped=stats.skipped, - artifact_data=stats.custom.get('_run', {}) if hasattr(stats, 'custom') else {} + skipped=stats.skipped ) + # write custom set_stat artifact data to the local disk so that it can + # be persisted by awx after the process exits + custom_artifact_data = stats.custom.get('_run', {}) if hasattr(stats, 'custom') else {} + if custom_artifact_data: + # create the directory for custom stats artifacts to live in (if it doesn't exist) + custom_artifacts_dir = os.path.join(os.getenv('AWX_PRIVATE_DATA_DIR'), 'artifacts') + os.makedirs(custom_artifacts_dir, mode=stat.S_IXUSR + stat.S_IWUSR + stat.S_IRUSR) + + custom_artifacts_path = os.path.join(custom_artifacts_dir, 'custom') + with codecs.open(custom_artifacts_path, 'w', encoding='utf-8') as f: + os.chmod(custom_artifacts_path, stat.S_IRUSR | stat.S_IWUSR) + json.dump(custom_artifact_data, f) + with self.capture_event_data('playbook_on_stats', **event_data): super(BaseCallbackModule, self).v2_playbook_on_stats(stats) diff --git a/awx/lib/tests/test_display_callback.py b/awx/lib/tests/test_display_callback.py index fefd5d4188..e87f3ec306 100644 --- a/awx/lib/tests/test_display_callback.py +++ b/awx/lib/tests/test_display_callback.py @@ -2,7 +2,9 @@ from collections import OrderedDict import json import mock import os +import shutil import sys +import tempfile import pytest @@ -254,3 +256,26 @@ def test_callback_plugin_strips_task_environ_variables(executor, cache, playbook assert len(cache) for event in cache.values(): assert os.environ['PATH'] not in json.dumps(event) + + +@pytest.mark.parametrize('playbook', [ +{'custom_set_stat.yml': ''' +- name: custom set_stat calls should persist to the local disk so awx can save them + connection: local + hosts: all + tasks: + - set_stats: + data: + foo: "bar" +'''}, # noqa +]) +def test_callback_plugin_saves_custom_stats(executor, cache, playbook): + try: + private_data_dir = tempfile.mkdtemp() + with mock.patch.dict(os.environ, {'AWX_PRIVATE_DATA_DIR': private_data_dir}): + executor.run() + artifacts_path = os.path.join(private_data_dir, 'artifacts', 'custom') + with open(artifacts_path, 'r') as f: + assert json.load(f) == {'foo': 'bar'} + finally: + shutil.rmtree(os.path.join(private_data_dir)) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 3345dada44..9b04d368b8 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -659,6 +659,7 @@ class BaseTask(LogErrorsTask): # Derived class should call add_ansible_venv() or add_awx_venv() if self.should_use_proot(instance, **kwargs): env['PROOT_TMP_DIR'] = settings.AWX_PROOT_BASE_PATH + env['AWX_PRIVATE_DATA_DIR'] = kwargs['private_data_dir'] return env def build_safe_env(self, env, **kwargs): @@ -1307,6 +1308,21 @@ class RunJob(BaseTask): kwargs['private_data_dir'], kwargs['fact_modification_times'] ) + + # persist artifacts set via `set_stat` (if any) + custom_stats_path = os.path.join(kwargs['private_data_dir'], 'artifacts', 'custom') + if os.path.exists(custom_stats_path): + with open(custom_stats_path, 'r') as f: + custom_stat_data = None + try: + custom_stat_data = json.load(f) + except ValueError: + logger.warning('Could not parse custom `set_fact` data for job {}'.format(job.id)) + + if custom_stat_data: + job.artifacts = custom_stat_data + job.save(update_fields=['artifacts']) + try: inventory = job.inventory except Inventory.DoesNotExist: From aa469d730ea825119d2d5536e8255ab4632c7829 Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Wed, 24 Jan 2018 13:45:03 +0000 Subject: [PATCH 08/16] Wait for Slack RTM API websocket connection to be established --- awx/main/notifications/slack_backend.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/awx/main/notifications/slack_backend.py b/awx/main/notifications/slack_backend.py index 3cea4bd44e..6e966e882b 100644 --- a/awx/main/notifications/slack_backend.py +++ b/awx/main/notifications/slack_backend.py @@ -1,6 +1,7 @@ # Copyright (c) 2016 Ansible, Inc. # All Rights Reserved. +import time import logging from slackclient import SlackClient @@ -9,6 +10,7 @@ from django.utils.translation import ugettext_lazy as _ from awx.main.notifications.base import AWXBaseEmailBackend logger = logging.getLogger('awx.main.notifications.slack_backend') +WEBSOCKET_TIMEOUT = 30 class SlackBackend(AWXBaseEmailBackend): @@ -30,7 +32,18 @@ class SlackBackend(AWXBaseEmailBackend): if not self.connection.rtm_connect(): if not self.fail_silently: raise Exception("Slack Notification Token is invalid") - return True + + start = time.time() + time.clock() + elapsed = 0 + while elapsed < WEBSOCKET_TIMEOUT: + events = self.connection.rtm_read() + if any(event['type'] == 'hello' for event in events): + return True + elapsed = time.time() - start + time.sleep(0.5) + + raise RuntimeError("Slack Notification unable to establish websocket connection after {} seconds".format(WEBSOCKET_TIMEOUT)) def close(self): if self.connection is None: From 4c79e6912ec120e1bcb0ef70078a655c6192fe67 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Sun, 28 Jan 2018 21:18:51 -0500 Subject: [PATCH 09/16] bump templates form credential_types page limit --- awx/ui/client/src/shared/credentialTypesLookup.factory.js | 4 ++-- .../multi-credential/multi-credential-modal.directive.js | 2 +- .../multi-credential/multi-credential.service.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/awx/ui/client/src/shared/credentialTypesLookup.factory.js b/awx/ui/client/src/shared/credentialTypesLookup.factory.js index 74251d9aa0..c7da2c1d13 100644 --- a/awx/ui/client/src/shared/credentialTypesLookup.factory.js +++ b/awx/ui/client/src/shared/credentialTypesLookup.factory.js @@ -1,8 +1,8 @@ export default ['Rest', 'GetBasePath', 'ProcessErrors', function(Rest, GetBasePath, ProcessErrors) { - return function() { + return function(params = null) { Rest.setUrl(GetBasePath('credential_types')); - return Rest.get() + return Rest.get({ params }) .then(({data}) => { var val = {}; data.results.forEach(type => { diff --git a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.directive.js b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.directive.js index 6b9e8ba709..718537b218 100644 --- a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.directive.js +++ b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.directive.js @@ -10,7 +10,7 @@ export default ['templateUrl', 'Rest', 'GetBasePath', 'generateList', '$compile' templateUrl: templateUrl('templates/job_templates/multi-credential/multi-credential-modal'), link: function(scope, element) { - credentialTypesLookup() + credentialTypesLookup({ page_size: 200 }) .then(kinds => { scope.credentialKinds = kinds; diff --git a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js index a22937fb61..d8fa9385d5 100644 --- a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js +++ b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js @@ -77,7 +77,7 @@ export default ['Rest', 'ProcessErrors', '$q', 'GetBasePath', function(Rest, Pro // credential type selector val.getCredentialTypes = () => { Rest.setUrl(GetBasePath('credential_types')); - return Rest.get() + return Rest.get({ params: { page_size: 200 }}) .then(({data}) => { let credential_types = {}, credentialTypeOptions = []; From 982539f4442b3003ba741af9dd30d0ef711a86f4 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 29 Jan 2018 12:01:55 -0500 Subject: [PATCH 10/16] fix a bug when testing UDP-based logging configuration see: https://github.com/ansible/ansible-tower/issues/7868 --- awx/conf/views.py | 8 ++++++-- awx/main/utils/handlers.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/awx/conf/views.py b/awx/conf/views.py index 524abf476a..7da93ce7e2 100644 --- a/awx/conf/views.py +++ b/awx/conf/views.py @@ -21,7 +21,7 @@ from awx.api.generics import * # noqa from awx.api.permissions import IsSuperUser from awx.api.versioning import reverse, get_request_version from awx.main.utils import * # noqa -from awx.main.utils.handlers import BaseHTTPSHandler, LoggingConnectivityException +from awx.main.utils.handlers import BaseHTTPSHandler, UDPHandler, LoggingConnectivityException from awx.main.tasks import handle_setting_changes from awx.conf.license import get_licensed_features from awx.conf.models import Setting @@ -202,7 +202,11 @@ class SettingLoggingTest(GenericAPIView): for k, v in serializer.validated_data.items(): setattr(mock_settings, k, v) mock_settings.LOG_AGGREGATOR_LEVEL = 'DEBUG' - BaseHTTPSHandler.perform_test(mock_settings) + if mock_settings.LOG_AGGREGATOR_PROTOCOL.upper() == 'UDP': + UDPHandler.perform_test(mock_settings) + return Response(status=status.HTTP_201_CREATED) + else: + BaseHTTPSHandler.perform_test(mock_settings) except LoggingConnectivityException as e: return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) return Response(status=status.HTTP_200_OK) diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py index 20a30b499e..8ed1127292 100644 --- a/awx/main/utils/handlers.py +++ b/awx/main/utils/handlers.py @@ -292,6 +292,21 @@ class UDPHandler(BaseHandler): payload = _encode_payload_for_socket(payload) return self.socket.sendto(payload, (self._get_host(hostname_only=True), self.port or 0)) + @classmethod + def perform_test(cls, settings): + """ + Tests logging connectivity for the current logging settings. + """ + handler = cls.from_django_settings(settings) + handler.enabled_flag = True + handler.setFormatter(LogstashFormatter(settings_module=settings)) + logger = logging.getLogger(__file__) + fn, lno, func = logger.findCaller() + record = logger.makeRecord('awx', 10, fn, lno, + 'AWX Connection Test', tuple(), + None, func) + handler.emit(_encode_payload_for_socket(record)) + HANDLER_MAPPING = { 'https': BaseHTTPSHandler, From 28596b7d5e102c6bda5adcca65f5ea5fb6a92e18 Mon Sep 17 00:00:00 2001 From: John Mitchell Date: Tue, 30 Jan 2018 16:30:00 -0500 Subject: [PATCH 11/16] fix xss vulnerabilities - on host recent jobs popover - on schedule name tooltip --- .../shared/factories/set-status.factory.js | 2 +- awx/ui/client/src/scheduler/schedulerList.controller.js | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/awx/ui/client/src/inventories-hosts/shared/factories/set-status.factory.js b/awx/ui/client/src/inventories-hosts/shared/factories/set-status.factory.js index 3e7b76fb4a..aea5b5671f 100644 --- a/awx/ui/client/src/inventories-hosts/shared/factories/set-status.factory.js +++ b/awx/ui/client/src/inventories-hosts/shared/factories/set-status.factory.js @@ -60,7 +60,7 @@ export default html += "" + ellipsis(job.name) + "\n"; + ". Click for details\" data-placement=\"top\">" + $filter('sanitize')(ellipsis(job.name)) + "\n"; html += "\n"; } diff --git a/awx/ui/client/src/scheduler/schedulerList.controller.js b/awx/ui/client/src/scheduler/schedulerList.controller.js index fe6e884ce7..7b282e8e88 100644 --- a/awx/ui/client/src/scheduler/schedulerList.controller.js +++ b/awx/ui/client/src/scheduler/schedulerList.controller.js @@ -12,10 +12,10 @@ export default [ - '$scope', '$location', '$stateParams', 'ScheduleList', 'Rest', + '$filter', '$scope', '$location', '$stateParams', 'ScheduleList', 'Rest', 'rbacUiControlService', 'ToggleSchedule', 'DeleteSchedule', '$q', '$state', 'Dataset', 'ParentObject', 'UnifiedJobsOptions', - function($scope, $location, $stateParams, + function($filter, $scope, $location, $stateParams, ScheduleList, Rest, rbacUiControlService, ToggleSchedule, DeleteSchedule, @@ -90,7 +90,7 @@ export default [ schedule.status_tip = 'Schedule is stopped. Click to activate.'; } - schedule.nameTip = schedule.name; + schedule.nameTip = $filter('sanitize')(schedule.name); // include the word schedule if the schedule name does not include the word schedule if (schedule.name.indexOf("schedule") === -1 && schedule.name.indexOf("Schedule") === -1) { schedule.nameTip += " schedule"; @@ -99,7 +99,7 @@ export default [ if (job.name.indexOf("job") === -1 && job.name.indexOf("Job") === -1) { schedule.nameTip += "job "; } - schedule.nameTip += job.name; + schedule.nameTip += $filter('sanitize')(job.name); schedule.nameTip += ". Click to edit schedule."; } From 72715df7512f76bd9c9755868309712a1c1a7255 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 8 Feb 2018 15:12:08 -0500 Subject: [PATCH 12/16] fix a bug for "users should be able to change type of unused credential" see: https://github.com/ansible/ansible-tower/issues/7516 related: https://github.com/ansible/tower/pull/441 --- awx/api/serializers.py | 13 +++ .../tests/functional/api/test_credential.py | 99 +++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 7246cb1745..39ef37e829 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2192,6 +2192,19 @@ class CredentialSerializer(BaseSerializer): _('You cannot change the credential type of the credential, as it may break the functionality' ' of the resources using it.'), ) + + # TODO: When this code lands in awx, these relationships won't exist + # anymore, so we need to remove this code and make sure the related + # tests still pass. + for cls in (JobTemplate, Job): + for rel in ('extra_credentials__id', 'vault_credential_id'): + if cls.objects.filter(**{ + rel: self.instance.pk + }).count() > 0: + raise ValidationError( + _('You cannot change the credential type of the credential, as it may break the functionality' + ' of the resources using it.'), + ) return credential_type diff --git a/awx/main/tests/functional/api/test_credential.py b/awx/main/tests/functional/api/test_credential.py index e26447deea..60aa1dfd28 100644 --- a/awx/main/tests/functional/api/test_credential.py +++ b/awx/main/tests/functional/api/test_credential.py @@ -1480,6 +1480,105 @@ def test_credential_type_mutability(patch, organization, admin, credentialtype_s assert response.status_code == 200 +@pytest.mark.django_db +def test_vault_credential_type_mutability(patch, organization, admin, credentialtype_ssh, + credentialtype_vault): + cred = Credential( + credential_type=credentialtype_vault, + name='Best credential ever', + organization=organization, + inputs={ + 'vault_password': u'some-vault', + } + ) + cred.save() + + jt = JobTemplate(vault_credential=cred) + jt.save() + + def _change_credential_type(): + return patch( + reverse('api:credential_detail', kwargs={'version': 'v2', 'pk': cred.pk}), + { + 'credential_type': credentialtype_ssh.pk, + 'inputs': { + 'username': u'jim', + 'password': u'pass' + } + }, + admin + ) + + response = _change_credential_type() + assert response.status_code == 400 + expected = ['You cannot change the credential type of the credential, ' + 'as it may break the functionality of the resources using it.'] + assert response.data['credential_type'] == expected + + response = patch( + reverse('api:credential_detail', kwargs={'version': 'v2', 'pk': cred.pk}), + {'name': 'Worst credential ever'}, + admin + ) + assert response.status_code == 200 + assert Credential.objects.get(pk=cred.pk).name == 'Worst credential ever' + + jt.delete() + response = _change_credential_type() + assert response.status_code == 200 + + +@pytest.mark.django_db +def test_cloud_credential_type_mutability(patch, organization, admin, credentialtype_ssh, + credentialtype_aws): + cred = Credential( + credential_type=credentialtype_aws, + name='Best credential ever', + organization=organization, + inputs={ + 'username': u'jim', + 'password': u'pass' + } + ) + cred.save() + + jt = JobTemplate() + jt.save() + jt.extra_credentials.add(cred) + jt.save() + + def _change_credential_type(): + return patch( + reverse('api:credential_detail', kwargs={'version': 'v2', 'pk': cred.pk}), + { + 'credential_type': credentialtype_ssh.pk, + 'inputs': { + 'username': u'jim', + 'password': u'pass' + } + }, + admin + ) + + response = _change_credential_type() + assert response.status_code == 400 + expected = ['You cannot change the credential type of the credential, ' + 'as it may break the functionality of the resources using it.'] + assert response.data['credential_type'] == expected + + response = patch( + reverse('api:credential_detail', kwargs={'version': 'v2', 'pk': cred.pk}), + {'name': 'Worst credential ever'}, + admin + ) + assert response.status_code == 200 + assert Credential.objects.get(pk=cred.pk).name == 'Worst credential ever' + + jt.delete() + response = _change_credential_type() + assert response.status_code == 200 + + @pytest.mark.django_db @pytest.mark.parametrize('version, params', [ ['v1', { From 82e41b40bb35ed552d35beee67cd226cbbf3321f Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 12 Feb 2018 16:10:26 -0500 Subject: [PATCH 13/16] enforce strings for secret password inputs on Credentials see: https://github.com/ansible/ansible-tower/issues/7898 --- awx/main/fields.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/awx/main/fields.py b/awx/main/fields.py index 56c390aa8d..9e89d65dba 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -499,6 +499,12 @@ class CredentialInputField(JSONSchemaField): v != '$encrypted$', model_instance.pk ]): + if not isinstance(getattr(model_instance, k), six.string_types): + raise django_exceptions.ValidationError( + _('secret values must be of type string, not {}').format(type(v).__name__), + code='invalid', + params={'value': v}, + ) decrypted_values[k] = utils.decrypt_field(model_instance, k) else: decrypted_values[k] = v From 613d48cdbc2cebc0fe2336fd186e2dee73251807 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 13 Feb 2018 14:26:27 -0500 Subject: [PATCH 14/16] add support for new "BECOME" prompt in Ansible 2.5+ for adhoc commands see: https://github.com/ansible/ansible-tower/issues/7850 --- awx/main/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 9b04d368b8..6b9feeabe5 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2242,6 +2242,7 @@ class RunAdHocCommand(BaseTask): for method in PRIVILEGE_ESCALATION_METHODS: d[re.compile(r'%s password.*:\s*?$' % (method[0]), re.M)] = 'become_password' d[re.compile(r'%s password.*:\s*?$' % (method[0].upper()), re.M)] = 'become_password' + d[re.compile(r'BECOME password.*:\s*?$', re.M)] = 'become_password' d[re.compile(r'SSH password:\s*?$', re.M)] = 'ssh_password' d[re.compile(r'Password:\s*?$', re.M)] = 'ssh_password' return d From b01deb393ea865ff509ed0fc12fcf223ef9bb0c7 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 14 Feb 2018 12:29:02 -0500 Subject: [PATCH 15/16] use --export option for ansible-inventory --- awx/main/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 6b9feeabe5..dbc7d2ed30 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1892,6 +1892,8 @@ class RunInventoryUpdate(BaseTask): # Pass inventory source ID to inventory script. env['INVENTORY_SOURCE_ID'] = str(inventory_update.inventory_source_id) env['INVENTORY_UPDATE_ID'] = str(inventory_update.pk) + # Always use the --export option for ansible-inventory + env['ANSIBLE_INVENTORY_EXPORT'] = str(True) # Set environment variables specific to each source. # From 465e60546419337cea0885fb0f584a060b7058ea Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 15 Feb 2018 15:26:58 -0500 Subject: [PATCH 16/16] fix unicode bugs with log statements --- awx/main/tasks.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index dbc7d2ed30..52d70f51d3 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -17,6 +17,7 @@ import stat import tempfile import time import traceback +import six import urlparse import uuid from distutils.version import LooseVersion as Version @@ -1543,15 +1544,15 @@ class RunProjectUpdate(BaseTask): if not inv_src.update_on_project_update: continue if inv_src.scm_last_revision == scm_revision: - logger.debug('Skipping SCM inventory update for `{}` because ' - 'project has not changed.'.format(inv_src.name)) + logger.debug(six.text_type('Skipping SCM inventory update for `{}` because ' + 'project has not changed.').format(inv_src.name)) continue - logger.debug('Local dependent inventory update for `{}`.'.format(inv_src.name)) + logger.debug(six.text_type('Local dependent inventory update for `{}`.').format(inv_src.name)) with transaction.atomic(): if InventoryUpdate.objects.filter(inventory_source=inv_src, status__in=ACTIVE_STATES).exists(): - logger.info('Skipping SCM inventory update for `{}` because ' - 'another update is already active.'.format(inv_src.name)) + logger.info(six.text_type('Skipping SCM inventory update for `{}` because ' + 'another update is already active.').format(inv_src.name)) continue local_inv_update = inv_src.create_inventory_update( launch_type='scm',