Merge pull request #6910 from ryanpetrello/devel

merge in a few downstream fixes

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

awx/api/serializers.py

@@ -3884,15 +3884,23 @@ class ProjectUpdateEventSerializer(JobEventSerializer):
         return UriCleaner.remove_sensitive(obj.stdout)
 
     def get_event_data(self, obj):
-        try:
+        # the project update playbook uses the git, hg, or svn modules
-            return json.loads(
+        # to clone repositories, and those modules are prone to printing
-                UriCleaner.remove_sensitive(
+        # raw SCM URLs in their stdout (which *could* contain passwords)
-                    json.dumps(obj.event_data)
+        # attempt to detect and filter HTTP basic auth passwords in the stdout
+        # of these types of events
+        if obj.event_data.get('task_action') in ('git', 'hg', 'svn'):
+            try:
+                return json.loads(
+                    UriCleaner.remove_sensitive(
+                        json.dumps(obj.event_data)
+                    )
                 )
-            )
+            except Exception:
-        except Exception:
+                logger.exception("Failed to sanitize event_data")
-            logger.exception("Failed to sanitize event_data")
+                return {}
-            return {}
+        else:
+            return obj.event_data
 
 
 class AdHocCommandEventSerializer(BaseSerializer):

awx/main/dispatch/worker/base.py

@@ -8,6 +8,7 @@ import sys
 import redis
 import json
 import psycopg2
+import time
 from uuid import UUID
 from queue import Empty as QueueEmpty
 
@@ -116,18 +117,23 @@ class AWXConsumerRedis(AWXConsumerBase):
         super(AWXConsumerRedis, self).run(*args, **kwargs)
         self.worker.on_start()
 
-        queue = redis.Redis.from_url(settings.BROKER_URL)
+        time_to_sleep = 1
         while True:
-            try:
+            queue = redis.Redis.from_url(settings.BROKER_URL)
-                res = queue.blpop(self.queues)
+            while True:
-                res = json.loads(res[1])
+                try:
-                self.process_task(res)
+                    res = queue.blpop(self.queues)
-            except redis.exceptions.RedisError:
+                    time_to_sleep = 1
-                logger.exception("encountered an error communicating with redis")
+                    res = json.loads(res[1])
-            except (json.JSONDecodeError, KeyError):
+                    self.process_task(res)
-                logger.exception("failed to decode JSON message from redis")
+                except redis.exceptions.RedisError:
-            if self.should_stop:
+                    time_to_sleep = min(time_to_sleep * 2, 30)
-                return
+                    logger.exception(f"encountered an error communicating with redis. Reconnect attempt in {time_to_sleep} seconds")
+                    time.sleep(time_to_sleep)
+                except (json.JSONDecodeError, KeyError):
+                    logger.exception("failed to decode JSON message from redis")
+                if self.should_stop:
+                    return
 
 
 class AWXConsumerPG(AWXConsumerBase):

awx/main/tasks.py

@@ -1232,10 +1232,12 @@ class BaseTask(object):
             # this is a _little_ expensive to filter
             # with regex, but project updates don't have many events,
             # so it *should* have a negligible performance impact
+            task = event_data.get('event_data', {}).get('task_action')
             try:
-                event_data_json = json.dumps(event_data)
+                if task in ('git', 'hg', 'svn'):
-                event_data_json = UriCleaner.remove_sensitive(event_data_json)
+                    event_data_json = json.dumps(event_data)
-                event_data = json.loads(event_data_json)
+                    event_data_json = UriCleaner.remove_sensitive(event_data_json)
+                    event_data = json.loads(event_data_json)
             except json.JSONDecodeError:
                 pass
 

awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js

@@ -263,8 +263,8 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){
                 dataTitle: i18n._("Source Variables"),
                 dataPlacement: 'right',
                 awPopOver: i18n._(`Override variables found in openstack.yml and used by the inventory update script. For an example variable configuration
-                    <a href=\"https://github.com/ansible/ansible/blob/devel/contrib/inventory/openstack.yml\" target=\"_blank\">
+                    <a href=\"https://github.com/openstack/ansible-collections-openstack/blob/master/scripts/inventory/openstack.yml\" target=\"_blank\">
-                    view openstack.yml in the Ansible github repo.</a> Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`),
+                    view openstack.yml in the Openstack github repo.</a> Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`),
                 dataContainer: 'body',
                 subForm: 'sourceSubForm'
             },
@@ -280,8 +280,8 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){
                 dataTitle: i18n._("Source Variables"),
                 dataPlacement: 'right',
                 awPopOver: i18n._(`Override variables found in cloudforms.ini and used by the inventory update script. For an example variable configuration
-                    <a href=\"https://github.com/ansible/ansible/blob/devel/contrib/inventory/cloudforms.ini\" target=\"_blank\">
+                    <a href=\"https://github.com/ansible-collections/community.general/blob/master/scripts/inventory/cloudforms.ini\" target=\"_blank\">
-                    view cloudforms.ini in the Ansible github repo.</a> Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`),
+                    view cloudforms.ini in the Ansible Collections github repo.</a> Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`),
                 dataContainer: 'body',
                 subForm: 'sourceSubForm'
             },
@@ -297,8 +297,8 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){
                 dataTitle: i18n._("Source Variables"),
                 dataPlacement: 'right',
                 awPopOver: i18n._(`Override variables found in foreman.ini and used by the inventory update script. For an example variable configuration
-                    <a href=\"https://github.com/ansible/ansible/blob/devel/contrib/inventory/foreman.ini\" target=\"_blank\">
+                    <a href=\"https://github.com/ansible-collections/community.general/blob/master/scripts/inventory/foreman.ini\" target=\"_blank\">
-                    view foreman.ini in the Ansible github repo.</a> Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`),
+                    view foreman.ini in the Ansible Collections github repo.</a> Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`),
                 dataContainer: 'body',
                 subForm: 'sourceSubForm'
             },