From 2ecd055d1ef6ffa535b43604b492086403cba563 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Tue, 28 Apr 2020 12:24:34 -0400 Subject: [PATCH 1/6] sleep backoff on cb receiver reconnect * Sleep before trying to reconnect Most common reason for entering this reconnect loop is when Redis service stops before the callback receiver when stopping tower services. --- awx/main/dispatch/worker/base.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/awx/main/dispatch/worker/base.py b/awx/main/dispatch/worker/base.py index 74d2a6a6c3..3ff20609b6 100644 --- a/awx/main/dispatch/worker/base.py +++ b/awx/main/dispatch/worker/base.py @@ -8,6 +8,7 @@ import sys import redis import json import psycopg2 +import time from uuid import UUID from queue import Empty as QueueEmpty @@ -116,18 +117,23 @@ class AWXConsumerRedis(AWXConsumerBase): super(AWXConsumerRedis, self).run(*args, **kwargs) self.worker.on_start() - queue = redis.Redis.from_url(settings.BROKER_URL) + retry = 0 while True: - try: - res = queue.blpop(self.queues) - res = json.loads(res[1]) - self.process_task(res) - except redis.exceptions.RedisError: - logger.exception("encountered an error communicating with redis") - except (json.JSONDecodeError, KeyError): - logger.exception("failed to decode JSON message from redis") - if self.should_stop: - return + queue = redis.Redis.from_url(settings.BROKER_URL) + while True: + try: + res = queue.blpop(self.queues) + retry = 0 + res = json.loads(res[1]) + self.process_task(res) + except redis.exceptions.RedisError: + logger.exception(f"encountered an error communicating with redis. Reconnect attempt {retry}") + retry += 1 + time.sleep(min(retry * 2, 30)) + except (json.JSONDecodeError, KeyError): + logger.exception("failed to decode JSON message from redis") + if self.should_stop: + return class AWXConsumerPG(AWXConsumerBase): From 867475ad493481b61cc8294b896d7101f2aefaf0 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Tue, 28 Apr 2020 14:00:34 -0400 Subject: [PATCH 2/6] added in (what I believe to be) the correct links --- .../inventories/related/sources/sources.form.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js index 1423f881f2..2dda6bf73d 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js @@ -263,8 +263,8 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){ dataTitle: i18n._("Source Variables"), dataPlacement: 'right', awPopOver: i18n._(`Override variables found in openstack.yml and used by the inventory update script. For an example variable configuration - - view openstack.yml in the Ansible github repo. Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`), + + view openstack.yml in the Openstack github repo. Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`), dataContainer: 'body', subForm: 'sourceSubForm' }, @@ -280,8 +280,8 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){ dataTitle: i18n._("Source Variables"), dataPlacement: 'right', awPopOver: i18n._(`Override variables found in cloudforms.ini and used by the inventory update script. For an example variable configuration - - view cloudforms.ini in the Ansible github repo. Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`), + + view cloudforms.ini in the Ansible Collections github repo. Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`), dataContainer: 'body', subForm: 'sourceSubForm' }, @@ -297,8 +297,8 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){ dataTitle: i18n._("Source Variables"), dataPlacement: 'right', awPopOver: i18n._(`Override variables found in foreman.ini and used by the inventory update script. For an example variable configuration - - view foreman.ini in the Ansible github repo. Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`), + + view foreman.ini in the Ansible Collections github repo. Enter inventory variables using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.`), dataContainer: 'body', subForm: 'sourceSubForm' }, From a8f52c16397fc0c915a7d2af3bdb3ac261bccc62 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Tue, 28 Apr 2020 12:41:47 -0400 Subject: [PATCH 3/6] actually do exponential calc rather than *2 * Log the time til reconnect attemp to log message rather than attempt number --- awx/main/dispatch/worker/base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/awx/main/dispatch/worker/base.py b/awx/main/dispatch/worker/base.py index 3ff20609b6..c796e6162e 100644 --- a/awx/main/dispatch/worker/base.py +++ b/awx/main/dispatch/worker/base.py @@ -117,19 +117,19 @@ class AWXConsumerRedis(AWXConsumerBase): super(AWXConsumerRedis, self).run(*args, **kwargs) self.worker.on_start() - retry = 0 + time_to_sleep = 1 while True: queue = redis.Redis.from_url(settings.BROKER_URL) while True: try: res = queue.blpop(self.queues) - retry = 0 + time_to_sleep = 1 res = json.loads(res[1]) self.process_task(res) except redis.exceptions.RedisError: - logger.exception(f"encountered an error communicating with redis. Reconnect attempt {retry}") - retry += 1 - time.sleep(min(retry * 2, 30)) + time_to_sleep = min(time_to_sleep*2, 30) + logger.exception(f"encountered an error communicating with redis. Reconnect attempt in {time_to_sleep} seconds") + time.sleep(time_to_sleep) except (json.JSONDecodeError, KeyError): logger.exception("failed to decode JSON message from redis") if self.should_stop: From e4921abfff00c205a3842063c0fa26676cf4bd26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ely=C3=A9zer=20Rezende?= Date: Wed, 29 Apr 2020 13:55:35 -0400 Subject: [PATCH 4/6] Keep awxkit's requirements on the setup.py awxkit's setup.py was making use of pip internal structures to parse the requirements.txt file. This is not a good thing as they may change, actually that just happened. To avoid this in the future, move the list of requirements to setup.py and make requirements.txt list `.` as the only item. This way we keep a single place to update requirements in the future and avoid accessing pip's internals. --- awxkit/requirements.txt | 3 +-- awxkit/setup.py | 11 ++++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/awxkit/requirements.txt b/awxkit/requirements.txt index 6c9fdba970..9c558e357c 100644 --- a/awxkit/requirements.txt +++ b/awxkit/requirements.txt @@ -1,2 +1 @@ -PyYAML -requests +. diff --git a/awxkit/setup.py b/awxkit/setup.py index 2f304f2600..7dac7a7b8e 100644 --- a/awxkit/setup.py +++ b/awxkit/setup.py @@ -2,12 +2,6 @@ import os import glob import shutil from setuptools import setup, find_packages, Command -try: # for pip >= 10 - from pip._internal.req import parse_requirements -except ImportError: # for pip <= 9.0.3 - from pip.req import parse_requirements - -requirements = [str(r.req) for r in parse_requirements('requirements.txt', session=False)] def get_version(): @@ -66,7 +60,10 @@ setup( 'clean': CleanCommand, }, include_package_data=True, - install_requires=requirements, + install_requires=[ + 'PyYAML', + 'requests', + ], python_requires=">=3.6", extras_require={ 'formatting': ['jq'], From bf65b40241d60e99e49b3109a33c26f26cd418ec Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 1 May 2020 09:58:25 -0400 Subject: [PATCH 5/6] only sanitize project update events for the scm modules these are the only modules in the project update playbook that actually utilize the SCM URL (which is what potentially contains sensitive data) --- awx/api/serializers.py | 24 ++++++++++++++++-------- awx/main/tasks.py | 8 +++++--- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 3960790ce7..d8152adb34 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -3899,15 +3899,23 @@ class ProjectUpdateEventSerializer(JobEventSerializer): return UriCleaner.remove_sensitive(obj.stdout) def get_event_data(self, obj): - try: - return json.loads( - UriCleaner.remove_sensitive( - json.dumps(obj.event_data) + # the project update playbook uses the git, hg, or svn modules + # to clone repositories, and those modules are prone to printing + # raw SCM URLs in their stdout (which *could* contain passwords) + # attempt to detect and filter HTTP basic auth passwords in the stdout + # of these types of events + if obj.event_data.get('task_action') in ('git', 'hg', 'svn'): + try: + return json.loads( + UriCleaner.remove_sensitive( + json.dumps(obj.event_data) + ) ) - ) - except Exception: - logger.exception("Failed to sanitize event_data") - return {} + except Exception: + logger.exception("Failed to sanitize event_data") + return {} + else: + return obj.event_data class AdHocCommandEventSerializer(BaseSerializer): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index e8a5bf5a57..26e73f4d6e 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1232,10 +1232,12 @@ class BaseTask(object): # this is a _little_ expensive to filter # with regex, but project updates don't have many events, # so it *should* have a negligible performance impact + task = event_data.get('event_data', {}).get('task_action') try: - event_data_json = json.dumps(event_data) - event_data_json = UriCleaner.remove_sensitive(event_data_json) - event_data = json.loads(event_data_json) + if task in ('git', 'hg', 'svn'): + event_data_json = json.dumps(event_data) + event_data_json = UriCleaner.remove_sensitive(event_data_json) + event_data = json.loads(event_data_json) except json.JSONDecodeError: pass From b4b261b918c7f18f835295d492cd59782755a93b Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 1 May 2020 13:51:37 -0400 Subject: [PATCH 6/6] fix busted flake8 --- awx/main/dispatch/worker/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/dispatch/worker/base.py b/awx/main/dispatch/worker/base.py index c796e6162e..b0611676fa 100644 --- a/awx/main/dispatch/worker/base.py +++ b/awx/main/dispatch/worker/base.py @@ -127,7 +127,7 @@ class AWXConsumerRedis(AWXConsumerBase): res = json.loads(res[1]) self.process_task(res) except redis.exceptions.RedisError: - time_to_sleep = min(time_to_sleep*2, 30) + time_to_sleep = min(time_to_sleep * 2, 30) logger.exception(f"encountered an error communicating with redis. Reconnect attempt in {time_to_sleep} seconds") time.sleep(time_to_sleep) except (json.JSONDecodeError, KeyError):