From 7eb74cd01124dc28cb5cf2c58ee62e90a6b7173b Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Wed, 24 Feb 2016 09:51:12 -0500 Subject: [PATCH 01/12] Bump 2.4.5 version, changelogs, and reprepo --- awx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/__init__.py b/awx/__init__.py index 72b8f62cbf..63561718b3 100644 --- a/awx/__init__.py +++ b/awx/__init__.py @@ -6,7 +6,7 @@ import sys import warnings import site -__version__ = '2.4.4' +__version__ = '2.4.5' __all__ = ['__version__'] From df3715a36f5637bb777dd043fb18311ff55d0a0e Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Wed, 24 Feb 2016 11:35:24 -0500 Subject: [PATCH 02/12] Fix for tasks breaking when using 'yum' with ansible 1.9.4 Fixes #1019 --- awx/plugins/callback/job_event_callback.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/awx/plugins/callback/job_event_callback.py b/awx/plugins/callback/job_event_callback.py index 041ce5bc53..471a95f4cd 100644 --- a/awx/plugins/callback/job_event_callback.py +++ b/awx/plugins/callback/job_event_callback.py @@ -2,10 +2,10 @@ # This file is a utility Ansible plugin that is not part of the AWX or Ansible # packages. It does not import any code from either package, nor does its # license apply to Ansible or AWX. -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: -# +# # Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. # @@ -66,8 +66,12 @@ CENSOR_FIELD_WHITELIST=[ 'skip_reason', ] -def censor(obj): - if obj.get('_ansible_no_log', False): +def censor(obj, no_log=False): + if type(obj) is not dict: + if no_log: + return "the output has been hidden due to the fact that 'no_log: true' was specified for this result" + return obj + if obj.get('_ansible_no_log', no_log): new_obj = {} for k in CENSOR_FIELD_WHITELIST: if k in obj: @@ -80,8 +84,12 @@ def censor(obj): new_obj['censored'] = "the output has been hidden due to the fact that 'no_log: true' was specified for this result" obj = new_obj if 'results' in obj: - for i in xrange(len(obj['results'])): - obj['results'][i] = censor(obj['results'][i]) + if type(obj['results']) is list: + for i in xrange(len(obj['results'])): + obj['results'][i] = censor(obj['results'][i], obj.get('_ansible_no_log', no_log)) + elif obj.get('_ansible_no_log', False): + obj['results'] = "the output has been hidden due to the fact that 'no_log: true' was specified for this result" + return obj @@ -410,7 +418,7 @@ class JobCallbackModule(BaseCallbackModule): # this from a normal task self._log_event('playbook_on_task_start', task=task, name=task.get_name()) - + def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None): From f08aeeda47e02170eb3ab88024a1e30bac1f0adc Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Wed, 24 Feb 2016 15:56:43 -0500 Subject: [PATCH 03/12] Backporting test fixes from PR #1020: Fix error with ad hoc command events when running in check mode. Backport of the test fixes from PR #1020 to 2.4.5, this fixes the unit tests around our fix for #1019 (hopefully) --- awx/main/models/ad_hoc_commands.py | 5 +++-- awx/main/tests/ad_hoc.py | 8 ++++---- awx/plugins/callback/job_event_callback.py | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/awx/main/models/ad_hoc_commands.py b/awx/main/models/ad_hoc_commands.py index 80520cdb1e..1371a39c37 100644 --- a/awx/main/models/ad_hoc_commands.py +++ b/awx/main/models/ad_hoc_commands.py @@ -218,8 +218,9 @@ class AdHocCommandEvent(CreatedModifiedModel): ('runner_on_unreachable', _('Host Unreachable'), True), # Tower won't see no_hosts (check is done earlier without callback). #('runner_on_no_hosts', _('No Hosts Matched'), False), - # Tower should probably never see skipped (no conditionals). - #('runner_on_skipped', _('Host Skipped'), False), + # Tower will see skipped (when running in check mode for a module that + # does not support check mode). + ('runner_on_skipped', _('Host Skipped'), False), # Tower does not support async for ad hoc commands. #('runner_on_async_poll', _('Host Polling'), False), #('runner_on_async_ok', _('Host Async OK'), False), diff --git a/awx/main/tests/ad_hoc.py b/awx/main/tests/ad_hoc.py index 957cd7c084..2b9cfe6866 100644 --- a/awx/main/tests/ad_hoc.py +++ b/awx/main/tests/ad_hoc.py @@ -123,8 +123,8 @@ class RunAdHocCommandTest(BaseAdHocCommandTest): self.assertFalse(ad_hoc_command.passwords_needed_to_start) self.assertTrue(ad_hoc_command.signal_start()) ad_hoc_command = AdHocCommand.objects.get(pk=ad_hoc_command.pk) - self.check_job_result(ad_hoc_command, 'failed') - self.check_ad_hoc_command_events(ad_hoc_command, 'unreachable') + self.check_job_result(ad_hoc_command, 'successful') + self.check_ad_hoc_command_events(ad_hoc_command, 'skipped') @mock.patch('awx.main.tasks.BaseTask.run_pexpect', return_value=('canceled', 0)) def test_cancel_ad_hoc_command(self, ignore): @@ -568,7 +568,7 @@ class AdHocCommandApiTest(BaseAdHocCommandTest): with self.current_user('admin'): response = self.run_test_ad_hoc_command(become_enabled=True) self.assertEqual(response['become_enabled'], True) - + # Try to run with expired license. self.create_expired_license_file() with self.current_user('admin'): @@ -1199,7 +1199,7 @@ class AdHocCommandApiTest(BaseAdHocCommandTest): with self.current_user('admin'): response = self.run_test_ad_hoc_command() - # Test the ad hoc command events list for a host. Should return the + # Test the ad hoc command events list for a host. Should return the # events only for that particular host. url = reverse('api:host_ad_hoc_command_events_list', args=(self.host.pk,)) with self.current_user('admin'): diff --git a/awx/plugins/callback/job_event_callback.py b/awx/plugins/callback/job_event_callback.py index 471a95f4cd..be747bc755 100644 --- a/awx/plugins/callback/job_event_callback.py +++ b/awx/plugins/callback/job_event_callback.py @@ -487,6 +487,7 @@ class AdHocCommandCallbackModule(BaseCallbackModule): def __init__(self): self.ad_hoc_command_id = int(os.getenv('AD_HOC_COMMAND_ID', '0')) self.rest_api_path = '/api/v1/ad_hoc_commands/%d/events/' % self.ad_hoc_command_id + self.skipped_hosts = set() super(AdHocCommandCallbackModule, self).__init__() def _log_event(self, event, **event_data): @@ -497,6 +498,19 @@ class AdHocCommandCallbackModule(BaseCallbackModule): def runner_on_file_diff(self, host, diff): pass # Ignore file diff for ad hoc commands. + def runner_on_ok(self, host, res): + # When running in check mode using a module that does not support check + # mode, Ansible v1.9 will call runner_on_skipped followed by + # runner_on_ok for the same host; only capture the skipped event and + # ignore the ok event. + if host not in self.skipped_hosts: + super(AdHocCommandCallbackModule, self).runner_on_ok(host, res) + + def runner_on_skipped(self, host, item=None): + super(AdHocCommandCallbackModule, self).runner_on_skipped(host, item) + self.skipped_hosts.add(host) + + if os.getenv('JOB_ID', ''): CallbackModule = JobCallbackModule From 519a1911eabb4f37ed114edfa8e4f6925c0bc004 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Wed, 24 Feb 2016 18:02:20 -0500 Subject: [PATCH 04/12] Use better isinstance(x) type checking --- awx/plugins/callback/job_event_callback.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/plugins/callback/job_event_callback.py b/awx/plugins/callback/job_event_callback.py index be747bc755..eb2f3e9d6e 100644 --- a/awx/plugins/callback/job_event_callback.py +++ b/awx/plugins/callback/job_event_callback.py @@ -67,7 +67,7 @@ CENSOR_FIELD_WHITELIST=[ ] def censor(obj, no_log=False): - if type(obj) is not dict: + if not isinstance(obj, dict): if no_log: return "the output has been hidden due to the fact that 'no_log: true' was specified for this result" return obj @@ -84,7 +84,7 @@ def censor(obj, no_log=False): new_obj['censored'] = "the output has been hidden due to the fact that 'no_log: true' was specified for this result" obj = new_obj if 'results' in obj: - if type(obj['results']) is list: + if isinstance(obj['results'], list): for i in xrange(len(obj['results'])): obj['results'][i] = censor(obj['results'][i], obj.get('_ansible_no_log', no_log)) elif obj.get('_ansible_no_log', False): From 26d11b2fb74ff38805b159e640b97413a2510e8f Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Thu, 25 Feb 2016 11:23:00 -0500 Subject: [PATCH 05/12] Default play names to the hosts the play was run against Fixes #1030 --- awx/plugins/callback/job_event_callback.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/awx/plugins/callback/job_event_callback.py b/awx/plugins/callback/job_event_callback.py index eb2f3e9d6e..b43fa4c998 100644 --- a/awx/plugins/callback/job_event_callback.py +++ b/awx/plugins/callback/job_event_callback.py @@ -463,6 +463,10 @@ class JobCallbackModule(BaseCallbackModule): def v2_playbook_on_play_start(self, play): setattr(self, 'play', play) + # Ansible 2.0.0.2 doesn't default .name to hosts like it did in 1.9.4, + # though that default will likely return in a future version of Ansible. + if not hasattr(play, 'name') or play.name == '': + play.name = ', '.join(play.hosts) self._log_event('playbook_on_play_start', name=play.name, pattern=play.hosts) From 5dc164bb762879689daaa06fc4ac76cb77eb1948 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Thu, 25 Feb 2016 11:38:09 -0500 Subject: [PATCH 06/12] Handle both string and list hosts in our hosts->name conversion Base our play.name fix off of what Ansible will be doing in the next release: https://github.com/ansible/ansible/commit/e2d2798a4238d95590777e8d41b9b28f324bd91d --- awx/plugins/callback/job_event_callback.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/awx/plugins/callback/job_event_callback.py b/awx/plugins/callback/job_event_callback.py index b43fa4c998..5108f97c31 100644 --- a/awx/plugins/callback/job_event_callback.py +++ b/awx/plugins/callback/job_event_callback.py @@ -465,8 +465,11 @@ class JobCallbackModule(BaseCallbackModule): setattr(self, 'play', play) # Ansible 2.0.0.2 doesn't default .name to hosts like it did in 1.9.4, # though that default will likely return in a future version of Ansible. - if not hasattr(play, 'name') or play.name == '': - play.name = ', '.join(play.hosts) + if (not hasattr(play, 'name') or not play.name) and hasattr(play, 'hosts'): + if isinstance(play.hosts, list): + play.name = ','.join(play.hosts) + else: + play.name = play.hosts self._log_event('playbook_on_play_start', name=play.name, pattern=play.hosts) From 24f7194648a03e73e253a154bb002fca89a871a2 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 26 Feb 2016 09:23:41 -0500 Subject: [PATCH 07/12] fix case of Ansible v2 _result.cmd is list for release 2.4.5 --- awx/plugins/callback/job_event_callback.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx/plugins/callback/job_event_callback.py b/awx/plugins/callback/job_event_callback.py index 5108f97c31..430ae4c80d 100644 --- a/awx/plugins/callback/job_event_callback.py +++ b/awx/plugins/callback/job_event_callback.py @@ -77,6 +77,8 @@ def censor(obj, no_log=False): if k in obj: new_obj[k] = obj[k] if k == 'cmd' and k in obj: + if isinstance(obj['cmd'], list): + obj['cmd'] = ' '.join(obj['cmd']) if re.search(r'\s', obj['cmd']): new_obj['cmd'] = re.sub(r'^(([^\s\\]|\\\s)+).*$', r'\1 ', From 48d3aa2bc1c03de5789b0f05ad0fe7ce4edaa555 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 3 Mar 2016 17:17:13 -0500 Subject: [PATCH 08/12] bump boto --- requirements/requirements.txt | 2 +- requirements/requirements_python26.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index d11279e736..3e47e5e0ac 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -6,7 +6,7 @@ appdirs==1.4.0 azure==0.9.0 Babel==2.2.0 billiard==3.3.0.16 -boto==2.34.0 +boto==2.39.0 celery==3.1.10 cffi==1.5.0 cliff==1.15.0 diff --git a/requirements/requirements_python26.txt b/requirements/requirements_python26.txt index 8158d952df..c7aab9b249 100644 --- a/requirements/requirements_python26.txt +++ b/requirements/requirements_python26.txt @@ -7,7 +7,7 @@ argparse==1.2.1 azure==0.9.0 Babel==1.3 billiard==3.3.0.16 -boto==2.34.0 +boto==2.39.0 celery==3.1.10 cffi==1.1.2 cliff==1.13.0 From f2284a00988d7b10cc243a3aa6d9d3c43017d3a2 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 22 Mar 2016 12:27:23 -0400 Subject: [PATCH 09/12] Handle runner items from ansible v2 Also denote whether the trailing runner_on_ was a loop event --- .../commands/run_callback_receiver.py | 2 +- awx/plugins/callback/job_event_callback.py | 23 +++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/awx/main/management/commands/run_callback_receiver.py b/awx/main/management/commands/run_callback_receiver.py index cccf07b4db..6ba078241f 100644 --- a/awx/main/management/commands/run_callback_receiver.py +++ b/awx/main/management/commands/run_callback_receiver.py @@ -134,7 +134,7 @@ class CallbackReceiver(object): 'playbook_on_import_for_host', 'playbook_on_not_import_for_host'): parent = job_parent_events.get('playbook_on_play_start', None) - elif message['event'].startswith('runner_on_'): + elif message['event'].startswith('runner_on_') or message['event'].startswith('runner_item_on_'): list_parents = [] list_parents.append(job_parent_events.get('playbook_on_setup', None)) list_parents.append(job_parent_events.get('playbook_on_task_start', None)) diff --git a/awx/plugins/callback/job_event_callback.py b/awx/plugins/callback/job_event_callback.py index 430ae4c80d..e55c91e4a6 100644 --- a/awx/plugins/callback/job_event_callback.py +++ b/awx/plugins/callback/job_event_callback.py @@ -175,7 +175,6 @@ class BaseCallbackModule(object): self._init_connection() if self.context is None: self._start_connection() - self.socket.send_json(msg) self.socket.recv() return @@ -224,16 +223,19 @@ class BaseCallbackModule(object): ignore_errors=ignore_errors) def v2_runner_on_failed(self, result, ignore_errors=False): + event_is_loop = result._task.loop if hasattr(result._task, 'loop') else None self._log_event('runner_on_failed', host=result._host.name, res=result._result, task=result._task, - ignore_errors=ignore_errors) + ignore_errors=ignore_errors, event_loop=event_is_loop) def runner_on_ok(self, host, res): self._log_event('runner_on_ok', host=host, res=res) def v2_runner_on_ok(self, result): + event_is_loop = result._task.loop if hasattr(result._task, 'loop') else None self._log_event('runner_on_ok', host=result._host.name, - task=result._task, res=result._result) + task=result._task, res=result._result, + event_loop=event_is_loop) def runner_on_error(self, host, msg): self._log_event('runner_on_error', host=host, msg=msg) @@ -245,8 +247,9 @@ class BaseCallbackModule(object): self._log_event('runner_on_skipped', host=host, item=item) def v2_runner_on_skipped(self, result): + event_is_loop = result._task.loop if hasattr(result._task, 'loop') else None self._log_event('runner_on_skipped', host=result._host.name, - task=result._task) + task=result._task, event_loop=event_is_loop) def runner_on_unreachable(self, host, res): self._log_event('runner_on_unreachable', host=host, res=res) @@ -280,6 +283,18 @@ class BaseCallbackModule(object): self._log_event('runner_on_file_diff', host=result._host.name, task=result._task, diff=diff) + def v2_runner_item_on_ok(self, result): + self._log_event('runner_item_on_ok', res=result._result, host=result._host.name, + task=result._task) + + def v2_runner_item_on_failed(self, result): + self._log_event('runner_item_on_failed', res=result._result, host=result._host.name, + task=result._task) + + def v2_runner_item_on_skipped(self, result): + self._log_event('runner_item_on_skipped', res=result._result, host=result._host.name, + task=result._task) + @staticmethod def terminate_ssh_control_masters(): # Determine if control persist is being used and if any open sockets From 9e3d4a99d2ce30733a384d61370e5397a4bd9819 Mon Sep 17 00:00:00 2001 From: mabashian Date: Wed, 23 Mar 2016 15:58:27 -0400 Subject: [PATCH 10/12] Added logic to not show the loop summary events in the UI by looking at event_data.event_loop. --- awx/ui/client/src/helpers/HostEventsViewer.js | 9 ++++++++- awx/ui/client/src/helpers/JobDetail.js | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/awx/ui/client/src/helpers/HostEventsViewer.js b/awx/ui/client/src/helpers/HostEventsViewer.js index e8fc5a940a..e68f82cada 100644 --- a/awx/ui/client/src/helpers/HostEventsViewer.js +++ b/awx/ui/client/src/helpers/HostEventsViewer.js @@ -3,7 +3,7 @@ * * All Rights Reserved *************************************************/ - + /** * @ngdoc function * @name helpers.function:HostEventsViewer @@ -273,6 +273,13 @@ export default .success(function(data) { var lastID; scope.hostViewSearching = false; + // Loop across the events and remove any events where + // event_data.event_loop is not null + for(var i=data.results.length-1; i>=0; i--){ + if(data.results[i].event_data && data.results[i].event_data.event_loop) { + data.results.splice(i, 1); + } + } if (data.results.length > 0) { lastID = data.results[data.results.length - 1].id; } diff --git a/awx/ui/client/src/helpers/JobDetail.js b/awx/ui/client/src/helpers/JobDetail.js index 3e54f37a5d..a151f655fb 100644 --- a/awx/ui/client/src/helpers/JobDetail.js +++ b/awx/ui/client/src/helpers/JobDetail.js @@ -1021,7 +1021,7 @@ export default } } - if (event.event !== "runner_on_no_hosts") { + if (event.event !== "runner_on_no_hosts" && (!event.event_data || (!event.event_data.event_loop || event.event_data.event_loop === null))) { scope.hostResults.push({ id: event.id, status: status, From 673c5929f9c379483ef44b211ea860618d30a176 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Mon, 11 Apr 2016 11:40:10 -0400 Subject: [PATCH 11/12] Improve the efficiency of the stdout dump database migration --- awx/main/migrations/0070_v221_changes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/main/migrations/0070_v221_changes.py b/awx/main/migrations/0070_v221_changes.py index 0bc36f27ac..039ff600ce 100644 --- a/awx/main/migrations/0070_v221_changes.py +++ b/awx/main/migrations/0070_v221_changes.py @@ -12,7 +12,7 @@ from django.conf import settings class Migration(DataMigration): def forwards(self, orm): - for j in orm.UnifiedJob.objects.filter(active=True): + for j in orm.UnifiedJob.objects.filter(active=True).only('id'): cur = connection.cursor() stdout_filename = os.path.join(settings.JOBOUTPUT_ROOT, "%d-%s.out" % (j.pk, str(uuid.uuid1()))) fd = open(stdout_filename, 'w') @@ -20,7 +20,7 @@ class Migration(DataMigration): fd.close() j.result_stdout_file = stdout_filename j.result_stdout_text = "" - j.save() + j.save(update_fields=['result_stdout_file', 'result_stdout_text']) sed_command = subprocess.Popen(["sed", "-i", "-e", "s/\\\\r\\\\n/\\n/g", stdout_filename]) sed_command.wait() From 9a9b198b2897b6a50188d2358886b5d3790b8735 Mon Sep 17 00:00:00 2001 From: Graham Mainwaring Date: Wed, 20 Apr 2016 10:13:26 -0400 Subject: [PATCH 12/12] Only export changed targets in reprepro --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 40eebbe644..b0940c7d85 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ DEBUILD_OPTS = --source-option="-I" DPUT_BIN ?= dput DPUT_OPTS ?= -c .dput.cf -u REPREPRO_BIN ?= reprepro -REPREPRO_OPTS ?= -b reprepro --export=force +REPREPRO_OPTS ?= -b reprepro --export=changed DEB_DIST ?= ifeq ($(OFFICIAL),yes) # Sign official builds