From 10d53637adc890f28a1189220b9e5c94cff225b7 Mon Sep 17 00:00:00 2001 From: Ben Thomasson Date: Fri, 23 Aug 2019 16:19:51 -0400 Subject: [PATCH 1/2] Changes uploader to use the insights api directly --- awx/main/analytics/core.py | 41 ++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/awx/main/analytics/core.py b/awx/main/analytics/core.py index 81a4814bdd..5c56c032d0 100644 --- a/awx/main/analytics/core.py +++ b/awx/main/analytics/core.py @@ -5,17 +5,17 @@ import os import os.path import tempfile import shutil -import subprocess +import requests from django.conf import settings -from django.utils.encoding import smart_str from django.utils.timezone import now, timedelta from rest_framework.exceptions import PermissionDenied from awx.conf.license import get_license -from awx.main.models import Job +from awx.main.models import Job, Credential from awx.main.access import access_registry from awx.main.models.ha import TowerAnalyticsState +from awx.main.utils import decrypt_field __all__ = ['register', 'gather', 'ship', 'table_version'] @@ -146,30 +146,23 @@ def gather(dest=None, module=None, collection_type='scheduled'): def ship(path): """ - Ship gathered metrics via the Insights agent + Ship gathered metrics via the Insights API """ try: - agent = 'insights-client' - if shutil.which(agent) is None: - logger.error('could not find {} on PATH'.format(agent)) - return logger.debug('shipping analytics file: {}'.format(path)) - try: - cmd = [ - agent, '--payload', path, '--content-type', settings.INSIGHTS_AGENT_MIME - ] - output = smart_str(subprocess.check_output(cmd, timeout=60 * 5)) - logger.debug(output) - # reset the `last_run` when data is shipped - run_now = now() - state = TowerAnalyticsState.get_solo() - state.last_run = run_now - state.save() - - except subprocess.CalledProcessError: - logger.exception('{} failure:'.format(cmd)) - except subprocess.TimeoutExpired: - logger.exception('{} timeout:'.format(cmd)) + url = settings.INSIGHTS_URL_BASE + '/api/ingress/v1/upload' + with open(path, 'rb') as f: + files = {'file': (os.path.basename(path), f, settings.INSIGHTS_AGENT_MIME)} + creds = Credential.objects.get(name=settings.INSIGHTS_URL_BASE) + response = requests.post(url, files=files, auth=(creds.inputs['username'], + decrypt_field(creds, 'password'))) + if response.status_code != 202: + logger.exception('Upload failure status {} text {}'.format(response.status_code, + response.text)) + run_now = now() + state = TowerAnalyticsState.get_solo() + state.last_run = run_now + state.save() finally: # cleanup tar.gz os.remove(path) From 6309c0a4262157d7373f6b190046508b2718cbbb Mon Sep 17 00:00:00 2001 From: Christian Adams Date: Thu, 29 Aug 2019 14:35:08 -0400 Subject: [PATCH 2/2] Upload using RH cred settings --- awx/main/analytics/core.py | 38 ++++++++++++++++++++++++++------------ awx/main/conf.py | 2 +- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/awx/main/analytics/core.py b/awx/main/analytics/core.py index 5c56c032d0..efcbfa4fa7 100644 --- a/awx/main/analytics/core.py +++ b/awx/main/analytics/core.py @@ -12,10 +12,9 @@ from django.utils.timezone import now, timedelta from rest_framework.exceptions import PermissionDenied from awx.conf.license import get_license -from awx.main.models import Job, Credential +from awx.main.models import Job from awx.main.access import access_registry from awx.main.models.ha import TowerAnalyticsState -from awx.main.utils import decrypt_field __all__ = ['register', 'gather', 'ship', 'table_version'] @@ -85,13 +84,12 @@ def gather(dest=None, module=None, collection_type='scheduled'): if last_run < max_interval or not last_run: last_run = max_interval - if _valid_license() is False: logger.exception("Invalid License provided, or No License Provided") return "Error: Invalid License provided, or No License Provided" - + if not settings.INSIGHTS_TRACKING_STATE: - logger.error("Insights analytics not enabled") + logger.error("Automation Analytics not enabled") return if module is None: @@ -146,19 +144,35 @@ def gather(dest=None, module=None, collection_type='scheduled'): def ship(path): """ - Ship gathered metrics via the Insights API + Ship gathered metrics to the Insights API """ + if not path: + logger.error('Automation Analytics TAR not found') + return + if "Error:" in str(path): + return try: logger.debug('shipping analytics file: {}'.format(path)) - url = settings.INSIGHTS_URL_BASE + '/api/ingress/v1/upload' + url = getattr(settings, 'AUTOMATION_ANALYTICS_URL', None) + if not url: + logger.error('AUTOMATION_ANALYTICS_URL is not set') + return + rh_user = getattr(settings, 'REDHAT_USERNAME', None) + rh_password = getattr(settings, 'REDHAT_PASSWORD', None) + if not rh_user: + return logger.error('REDHAT_USERNAME is not set') + if not rh_password: + return logger.error('REDHAT_PASSWORD is not set') with open(path, 'rb') as f: files = {'file': (os.path.basename(path), f, settings.INSIGHTS_AGENT_MIME)} - creds = Credential.objects.get(name=settings.INSIGHTS_URL_BASE) - response = requests.post(url, files=files, auth=(creds.inputs['username'], - decrypt_field(creds, 'password'))) + response = requests.post(url, + files=files, + verify=True, + auth=(rh_user, rh_password), + timeout=(31, 31)) if response.status_code != 202: - logger.exception('Upload failure status {} text {}'.format(response.status_code, - response.text)) + return logger.exception('Upload failed with status {}, {}'.format(response.status_code, + response.text)) run_now = now() state = TowerAnalyticsState.get_solo() state.last_run = run_now diff --git a/awx/main/conf.py b/awx/main/conf.py index 05529aadf9..b45c028a19 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -153,7 +153,7 @@ register( register( 'AUTOMATION_ANALYTICS_URL', field_class=fields.URLField, - default='https://cloud.redhat.com', + default='https://example.com', schemes=('http', 'https'), allow_plain_hostname=True, # Allow hostname only without TLD. label=_('Automation Analytics upload URL.'),