Merge branch 'release_2.4.5' into stable

* release_2.4.5:
  Remove distribute from the setup virtualenv if installed (#1631)
  Update changelogs for 2.4.5 release
  Only export changed targets in reprepro
  Improve the efficiency of the stdout dump database migration
  Added logic to not show the loop summary events in the UI by looking at event_data.event_loop.
  Handle runner items from ansible v2
  bump boto
  fix case of Ansible v2 _result.cmd is list for release 2.4.5
  Handle both string and list hosts in our hosts->name conversion
  Default play names to the hosts the play was run against
  Use better isinstance(x) type checking
  Backporting test fixes from PR #1020: Fix error with ad hoc command events when running in check mode.
  Fix for tasks breaking when using 'yum' with ansible 1.9.4
  Bump 2.4.5 version, changelogs, and reprepo

Makefile

@@ -76,7 +76,7 @@ DEBUILD_OPTS = --source-option="-I"
 DPUT_BIN ?= dput
 DPUT_OPTS ?= -c .dput.cf -u
 REPREPRO_BIN ?= reprepro
-REPREPRO_OPTS ?= -b reprepro --export=force
+REPREPRO_OPTS ?= -b reprepro --export=changed
 DEB_DIST ?=
 ifeq ($(OFFICIAL),yes)
     # Sign official builds

awx/__init__.py

@@ -6,7 +6,7 @@ import sys
 import warnings
 import site
 
-__version__ = '2.4.4'
+__version__ = '2.4.5'
 
 __all__ = ['__version__']
 

awx/main/management/commands/run_callback_receiver.py

@@ -134,7 +134,7 @@ class CallbackReceiver(object):
                                               'playbook_on_import_for_host',
                                               'playbook_on_not_import_for_host'):
                         parent = job_parent_events.get('playbook_on_play_start', None)
-                    elif message['event'].startswith('runner_on_'):
+                    elif message['event'].startswith('runner_on_') or message['event'].startswith('runner_item_on_'):
                         list_parents = []
                         list_parents.append(job_parent_events.get('playbook_on_setup', None))
                         list_parents.append(job_parent_events.get('playbook_on_task_start', None))

awx/main/migrations/0070_v221_changes.py

@@ -12,7 +12,7 @@ from django.conf import settings
 class Migration(DataMigration):
 
     def forwards(self, orm):
-        for j in orm.UnifiedJob.objects.filter(active=True):
+        for j in orm.UnifiedJob.objects.filter(active=True).only('id'):
             cur = connection.cursor()
             stdout_filename = os.path.join(settings.JOBOUTPUT_ROOT, "%d-%s.out" % (j.pk, str(uuid.uuid1())))
             fd = open(stdout_filename, 'w')
@@ -20,7 +20,7 @@ class Migration(DataMigration):
             fd.close()
             j.result_stdout_file = stdout_filename
             j.result_stdout_text = ""
-            j.save()
+            j.save(update_fields=['result_stdout_file', 'result_stdout_text'])
             sed_command = subprocess.Popen(["sed", "-i", "-e", "s/\\\\r\\\\n/\\n/g", stdout_filename])
             sed_command.wait()
 

awx/main/models/ad_hoc_commands.py

@@ -218,8 +218,9 @@ class AdHocCommandEvent(CreatedModifiedModel):
         ('runner_on_unreachable', _('Host Unreachable'), True),
         # Tower won't see no_hosts (check is done earlier without callback).
         #('runner_on_no_hosts', _('No Hosts Matched'), False),
-        # Tower should probably never see skipped (no conditionals).
-        #('runner_on_skipped', _('Host Skipped'), False),
+        # Tower will see skipped (when running in check mode for a module that
+        # does not support check mode).
+        ('runner_on_skipped', _('Host Skipped'), False),
         # Tower does not support async for ad hoc commands.
         #('runner_on_async_poll', _('Host Polling'), False),
         #('runner_on_async_ok', _('Host Async OK'), False),

awx/main/tests/ad_hoc.py

@@ -123,8 +123,8 @@ class RunAdHocCommandTest(BaseAdHocCommandTest):
         self.assertFalse(ad_hoc_command.passwords_needed_to_start)
         self.assertTrue(ad_hoc_command.signal_start())
         ad_hoc_command = AdHocCommand.objects.get(pk=ad_hoc_command.pk)
-        self.check_job_result(ad_hoc_command, 'failed')
-        self.check_ad_hoc_command_events(ad_hoc_command, 'unreachable')
+        self.check_job_result(ad_hoc_command, 'successful')
+        self.check_ad_hoc_command_events(ad_hoc_command, 'skipped')
 
     @mock.patch('awx.main.tasks.BaseTask.run_pexpect', return_value=('canceled', 0))
     def test_cancel_ad_hoc_command(self, ignore):
@@ -568,7 +568,7 @@ class AdHocCommandApiTest(BaseAdHocCommandTest):
         with self.current_user('admin'):
             response = self.run_test_ad_hoc_command(become_enabled=True)
             self.assertEqual(response['become_enabled'], True)
-        
+
         # Try to run with expired license.
         self.create_expired_license_file()
         with self.current_user('admin'):
@@ -1199,7 +1199,7 @@ class AdHocCommandApiTest(BaseAdHocCommandTest):
         with self.current_user('admin'):
             response = self.run_test_ad_hoc_command()
 
-        # Test the ad hoc command events list for a host.  Should return the 
+        # Test the ad hoc command events list for a host.  Should return the
         # events only for that particular host.
         url = reverse('api:host_ad_hoc_command_events_list', args=(self.host.pk,))
         with self.current_user('admin'):

awx/plugins/callback/job_event_callback.py

@@ -2,10 +2,10 @@
 # This file is a utility Ansible plugin that is not part of the AWX or Ansible
 # packages.  It does not import any code from either package, nor does its
 # license apply to Ansible or AWX.
-# 
+#
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
-# 
+#
 # Redistributions of source code must retain the above copyright notice, this
 # list of conditions and the following disclaimer.
 #
@@ -66,13 +66,19 @@ CENSOR_FIELD_WHITELIST=[
     'skip_reason',
 ]
 
-def censor(obj):
-    if obj.get('_ansible_no_log', False):
+def censor(obj, no_log=False):
+    if not isinstance(obj, dict):
+        if no_log:
+            return "the output has been hidden due to the fact that 'no_log: true' was specified for this result"
+        return obj
+    if obj.get('_ansible_no_log', no_log):
         new_obj = {}
         for k in CENSOR_FIELD_WHITELIST:
             if k in obj:
                 new_obj[k] = obj[k]
             if k == 'cmd' and k in obj:
+                if isinstance(obj['cmd'], list):
+                    obj['cmd'] = ' '.join(obj['cmd'])
                 if re.search(r'\s', obj['cmd']):
                     new_obj['cmd'] = re.sub(r'^(([^\s\\]|\\\s)+).*$',
                                             r'\1 <censored>',
@@ -80,8 +86,12 @@ def censor(obj):
         new_obj['censored'] = "the output has been hidden due to the fact that 'no_log: true' was specified for this result"
         obj = new_obj
     if 'results' in obj:
-        for i in xrange(len(obj['results'])):
-            obj['results'][i] = censor(obj['results'][i])
+        if isinstance(obj['results'], list):
+            for i in xrange(len(obj['results'])):
+                obj['results'][i] = censor(obj['results'][i], obj.get('_ansible_no_log', no_log))
+        elif obj.get('_ansible_no_log', False):
+            obj['results'] = "the output has been hidden due to the fact that 'no_log: true' was specified for this result"
+
     return obj
 
 
@@ -165,7 +175,6 @@ class BaseCallbackModule(object):
                     self._init_connection()
                 if self.context is None:
                     self._start_connection()
-
                 self.socket.send_json(msg)
                 self.socket.recv()
                 return
@@ -214,16 +223,19 @@ class BaseCallbackModule(object):
                         ignore_errors=ignore_errors)
 
     def v2_runner_on_failed(self, result, ignore_errors=False):
+        event_is_loop = result._task.loop if hasattr(result._task, 'loop') else None
         self._log_event('runner_on_failed', host=result._host.name,
                         res=result._result, task=result._task,
-                        ignore_errors=ignore_errors)
+                        ignore_errors=ignore_errors, event_loop=event_is_loop)
 
     def runner_on_ok(self, host, res):
         self._log_event('runner_on_ok', host=host, res=res)
 
     def v2_runner_on_ok(self, result):
+        event_is_loop = result._task.loop if hasattr(result._task, 'loop') else None
         self._log_event('runner_on_ok', host=result._host.name,
-                        task=result._task, res=result._result)
+                        task=result._task, res=result._result,
+                        event_loop=event_is_loop)
 
     def runner_on_error(self, host, msg):
         self._log_event('runner_on_error', host=host, msg=msg)
@@ -235,8 +247,9 @@ class BaseCallbackModule(object):
         self._log_event('runner_on_skipped', host=host, item=item)
 
     def v2_runner_on_skipped(self, result):
+        event_is_loop = result._task.loop if hasattr(result._task, 'loop') else None
         self._log_event('runner_on_skipped', host=result._host.name,
-                        task=result._task)
+                        task=result._task, event_loop=event_is_loop)
 
     def runner_on_unreachable(self, host, res):
         self._log_event('runner_on_unreachable', host=host, res=res)
@@ -270,6 +283,18 @@ class BaseCallbackModule(object):
         self._log_event('runner_on_file_diff', host=result._host.name,
                         task=result._task, diff=diff)
 
+    def v2_runner_item_on_ok(self, result):
+        self._log_event('runner_item_on_ok', res=result._result, host=result._host.name,
+                        task=result._task)
+
+    def v2_runner_item_on_failed(self, result):
+        self._log_event('runner_item_on_failed', res=result._result, host=result._host.name,
+                        task=result._task)
+
+    def v2_runner_item_on_skipped(self, result):
+        self._log_event('runner_item_on_skipped', res=result._result, host=result._host.name,
+                        task=result._task)
+
     @staticmethod
     def terminate_ssh_control_masters():
         # Determine if control persist is being used and if any open sockets
@@ -410,7 +435,7 @@ class JobCallbackModule(BaseCallbackModule):
         # this from a normal task
         self._log_event('playbook_on_task_start', task=task,
                         name=task.get_name())
-                        
+
     def playbook_on_vars_prompt(self, varname, private=True, prompt=None,
                                 encrypt=None, confirm=False, salt_size=None,
                                 salt=None, default=None):
@@ -455,6 +480,13 @@ class JobCallbackModule(BaseCallbackModule):
 
     def v2_playbook_on_play_start(self, play):
         setattr(self, 'play', play)
+        # Ansible 2.0.0.2 doesn't default .name to hosts like it did in 1.9.4,
+        # though that default will likely return in a future version of Ansible.
+        if (not hasattr(play, 'name') or not play.name) and hasattr(play, 'hosts'):
+            if isinstance(play.hosts, list):
+                play.name = ','.join(play.hosts)
+            else:
+                play.name = play.hosts
         self._log_event('playbook_on_play_start', name=play.name,
                         pattern=play.hosts)
 
@@ -479,6 +511,7 @@ class AdHocCommandCallbackModule(BaseCallbackModule):
     def __init__(self):
         self.ad_hoc_command_id = int(os.getenv('AD_HOC_COMMAND_ID', '0'))
         self.rest_api_path = '/api/v1/ad_hoc_commands/%d/events/' % self.ad_hoc_command_id
+        self.skipped_hosts = set()
         super(AdHocCommandCallbackModule, self).__init__()
 
     def _log_event(self, event, **event_data):
@@ -489,6 +522,19 @@ class AdHocCommandCallbackModule(BaseCallbackModule):
     def runner_on_file_diff(self, host, diff):
         pass # Ignore file diff for ad hoc commands.
 
+    def runner_on_ok(self, host, res):
+        # When running in check mode using a module that does not support check
+        # mode, Ansible v1.9 will call runner_on_skipped followed by
+        # runner_on_ok for the same host; only capture the skipped event and
+        # ignore the ok event.
+        if host not in self.skipped_hosts:
+            super(AdHocCommandCallbackModule, self).runner_on_ok(host, res)
+
+    def runner_on_skipped(self, host, item=None):
+        super(AdHocCommandCallbackModule, self).runner_on_skipped(host, item)
+        self.skipped_hosts.add(host)
+
+
 
 if os.getenv('JOB_ID', ''):
     CallbackModule = JobCallbackModule

awx/ui/client/src/helpers/HostEventsViewer.js

@@ -3,7 +3,7 @@
  *
  * All Rights Reserved
  *************************************************/
- 
+
    /**
  * @ngdoc function
  * @name helpers.function:HostEventsViewer
@@ -273,6 +273,13 @@ export default
                     .success(function(data) {
                         var lastID;
                         scope.hostViewSearching = false;
+                        // Loop across the events and remove any events where
+                        // event_data.event_loop is not null
+                        for(var i=data.results.length-1; i>=0; i--){
+                            if(data.results[i].event_data && data.results[i].event_data.event_loop) {
+                                data.results.splice(i, 1);
+                            }
+                        }
                         if (data.results.length > 0) {
                             lastID = data.results[data.results.length - 1].id;
                         }

awx/ui/client/src/helpers/JobDetail.js

@@ -1021,7 +1021,7 @@ export default
                                 }
                             }
 
-                            if (event.event !== "runner_on_no_hosts") {
+                            if (event.event !== "runner_on_no_hosts" && (!event.event_data || (!event.event_data.event_loop || event.event_data.event_loop === null))) {
                                 scope.hostResults.push({
                                     id: event.id,
                                     status: status,

requirements/requirements.txt

@@ -6,7 +6,7 @@ appdirs==1.4.0
 azure==0.9.0
 Babel==2.2.0
 billiard==3.3.0.16
-boto==2.34.0
+boto==2.39.0
 celery==3.1.10
 cffi==1.5.0
 cliff==1.15.0

requirements/requirements_python26.txt

@@ -7,7 +7,7 @@ argparse==1.2.1
 azure==0.9.0
 Babel==1.3
 billiard==3.3.0.16
-boto==2.34.0
+boto==2.39.0
 celery==3.1.10
 cffi==1.1.2
 cliff==1.13.0