From 8c1de7f1098fd6003543775e8fd776151d25a58e Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 8 Mar 2017 13:49:19 -0500 Subject: [PATCH] add an API endpoint for testing external log aggregrator connectivity see: #5164 --- awx/api/permissions.py | 10 ++- awx/conf/urls.py | 1 + awx/conf/views.py | 28 +++++++ .../tests/functional/api/test_settings.py | 60 +++++++++++++ awx/main/tests/unit/utils/test_handlers.py | 84 ++++++++++++++----- awx/main/utils/handlers.py | 30 +++++++ 6 files changed, 190 insertions(+), 23 deletions(-) diff --git a/awx/api/permissions.py b/awx/api/permissions.py index 8ec26a2cc8..966cf95ea5 100644 --- a/awx/api/permissions.py +++ b/awx/api/permissions.py @@ -16,7 +16,8 @@ from awx.main.utils import get_object_or_400 logger = logging.getLogger('awx.api.permissions') __all__ = ['ModelAccessPermission', 'JobTemplateCallbackPermission', - 'TaskPermission', 'ProjectUpdatePermission', 'UserPermission',] + 'TaskPermission', 'ProjectUpdatePermission', 'UserPermission', + 'IsSuperUser'] class ModelAccessPermission(permissions.BasePermission): @@ -208,3 +209,10 @@ class UserPermission(ModelAccessPermission): raise PermissionDenied() +class IsSuperUser(permissions.BasePermission): + """ + Allows access only to admin users. + """ + + def has_permission(self, request, view): + return request.user and request.user.is_superuser diff --git a/awx/conf/urls.py b/awx/conf/urls.py index 15505f4c3c..2c4f3ec91d 100644 --- a/awx/conf/urls.py +++ b/awx/conf/urls.py @@ -12,4 +12,5 @@ urlpatterns = patterns( 'awx.conf.views', url(r'^$', 'setting_category_list'), url(r'^(?P[a-z0-9-]+)/$', 'setting_singleton_detail'), + url(r'^logging/test/$', 'setting_logging_test'), ) diff --git a/awx/conf/views.py b/awx/conf/views.py index 99a3daab99..68b399444e 100644 --- a/awx/conf/views.py +++ b/awx/conf/views.py @@ -19,7 +19,9 @@ from rest_framework import status # Tower from awx.api.generics import * # noqa +from awx.api.permissions import IsSuperUser from awx.main.utils import * # noqa +from awx.main.utils.handlers import BaseHTTPSHandler, LoggingConnectivityException from awx.conf.license import get_licensed_features from awx.conf.models import Setting from awx.conf.serializers import SettingCategorySerializer, SettingSingletonSerializer @@ -130,6 +132,32 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView): settings.TOWER_URL_BASE = url +class SettingLoggingTest(GenericAPIView): + + view_name = _('Logging Connectivity Test') + model = Setting + serializer_class = SettingSingletonSerializer + permission_classes = (IsSuperUser,) + filter_backends = [] + new_in_320 = True + + def post(self, request, *args, **kwargs): + defaults = dict() + for key in settings_registry.get_registered_settings(category_slug='logging'): + try: + defaults[key] = settings_registry.get_setting_field(key).get_default() + except serializers.SkipField: + defaults[key] = None + obj = type('Settings', (object,), defaults)() + serializer = self.get_serializer(obj, data=request.data) + serializer.is_valid(raise_exception=True) + try: + BaseHTTPSHandler.perform_test(serializer.validated_data) + except LoggingConnectivityException as e: + return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + return Response(status=status.HTTP_200_OK) + + # Create view functions for all of the class-based views to simplify inclusion # in URL patterns and reverse URL lookups, converting CamelCase names to # lowercase_with_underscore (e.g. MyView.as_view() becomes my_view). diff --git a/awx/main/tests/functional/api/test_settings.py b/awx/main/tests/functional/api/test_settings.py index 6322f354e7..77b5294203 100644 --- a/awx/main/tests/functional/api/test_settings.py +++ b/awx/main/tests/functional/api/test_settings.py @@ -2,14 +2,19 @@ # All Rights Reserved. # Python +from collections import OrderedDict import pytest import os +# Mock +import mock + # Django from django.core.urlresolvers import reverse # AWX from awx.conf.models import Setting +from awx.main.utils.handlers import BaseHTTPSHandler, LoggingConnectivityException TEST_GIF_LOGO = 'data:image/gif;base64,R0lGODlhIQAjAPIAAP//////AP8AAMzMAJmZADNmAAAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQJCgAHACwAAAAAIQAjAAADo3i63P4wykmrvTjrzZsxXfR94WMQBFh6RECuixHMLyzPQ13ewZCvow9OpzEAjIBj79cJJmU+FceIVEZ3QRozxBttmyOBwPBtisdX4Bha3oxmS+llFIPHQXQKkiSEXz9PeklHBzx3hYNyEHt4fmmAhHp8Nz45KgV5FgWFOFEGmwWbGqEfniChohmoQZ+oqRiZDZhEgk81I4mwg4EKVbxzrDHBEAkAIfkECQoABwAsAAAAACEAIwAAA6V4utz+MMpJq724GpP15p1kEAQYQmOwnWjgrmxjuMEAx8rsDjZ+fJvdLWQAFAHGWo8FRM54JqIRmYTigDrDMqZTbbbMj0CgjTLHZKvPQH6CTx+a2vKR0XbbOsoZ7SphG057gjl+c0dGgzeGNiaBiSgbBQUHBV08NpOVlkMSk0FKjZuURHiiOJxQnSGfQJuoEKREejK0dFRGjoiQt7iOuLx0rgxYEQkAIfkECQoABwAsAAAAACEAIwAAA7h4utxnxslJDSGR6nrz/owxYB64QUEwlGaVqlB7vrAJscsd3Lhy+wBArGEICo3DUFH4QDqK0GMy51xOgcGlEAfJ+iAFie62chR+jYKaSAuQGOqwJp7jGQRDuol+F/jxZWsyCmoQfwYwgoM5Oyg1i2w0A2WQIW2TPYOIkleQmy+UlYygoaIPnJmapKmqKiusMmSdpjxypnALtrcHioq3ury7hGm3dnVosVpMWFmwREZbddDOSsjVswcJACH5BAkKAAcALAAAAAAhACMAAAOxeLrc/jDKSZUxNS9DCNYV54HURQwfGRlDEFwqdLVuGjOsW9/Odb0wnsUAKBKNwsMFQGwyNUHckVl8bqI4o43lA26PNkv1S9DtNuOeVirw+aTI3qWAQwnud1vhLSnQLS0GeFF+GoVKNF0fh4Z+LDQ6Bn5/MTNmL0mAl2E3j2aclTmRmYCQoKEDiaRDKFhJez6UmbKyQowHtzy1uEl8DLCnEktrQ2PBD1NxSlXKIW5hz6cJACH5BAkKAAcALAAAAAAhACMAAAOkeLrc/jDKSau9OOvNlTFd9H3hYxAEWDJfkK5LGwTq+g0zDR/GgM+10A04Cm56OANgqTRmkDTmSOiLMgFOTM9AnFJHuexzYBAIijZf2SweJ8ttbbXLmd5+wBiJosSCoGF/fXEeS1g8gHl9hxODKkh4gkwVIwUekESIhA4FlgV3PyCWG52WI2oGnR2lnUWpqhqVEF4Xi7QjhpsshpOFvLosrnpoEAkAIfkECQoABwAsAAAAACEAIwAAA6l4utz+MMpJq71YGpPr3t1kEAQXQltQnk8aBCa7bMMLy4wx1G8s072PL6SrGQDI4zBThCU/v50zCVhidIYgNPqxWZkDg0AgxB2K4vEXbBSvr1JtZ3uOext0x7FqovF6OXtfe1UzdjAxhINPM013ChtJER8FBQeVRX8GlpggFZWWfjwblTiigGZnfqRmpUKbljKxDrNMeY2eF4R8jUiSur6/Z8GFV2WBtwwJACH5BAkKAAcALAAAAAAhACMAAAO6eLrcZi3KyQwhkGpq8f6ONWQgaAxB8JTfg6YkO50pzD5xhaurhCsGAKCnEw6NucNDCAkyI8ugdAhFKpnJJdMaeiofBejowUseCr9GYa0j1GyMdVgjBxoEuPSZXWKf7gKBeHtzMms0gHgGfDIVLztmjScvNZEyk28qjT40b5aXlHCbDgOhnzedoqOOlKeopaqrCy56sgtotbYKhYW6e7e9tsHBssO6eSTIm1peV0iuFUZDyU7NJnmcuQsJACH5BAkKAAcALAAAAAAhACMAAAOteLrc/jDKSZsxNS9DCNYV54Hh4H0kdAXBgKaOwbYX/Miza1vrVe8KA2AoJL5gwiQgeZz4GMXlcHl8xozQ3kW3KTajL9zsBJ1+sV2fQfALem+XAlRApxu4ioI1UpC76zJ4fRqDBzI+LFyFhH1iiS59fkgziW07jjRAG5QDeECOLk2Tj6KjnZafW6hAej6Smgevr6yysza2tiCuMasUF2Yov2gZUUQbU8YaaqjLpQkAOw==' TEST_PNG_LOGO = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACEAAAAjCAYAAAAaLGNkAAAAAXNSR0IB2cksfwAAAdVpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6dGlmZj0iaHR0cDovL25zLmFkb2JlLmNvbS90aWZmLzEuMC8iPgogICAgICAgICA8dGlmZjpDb21wcmVzc2lvbj4xPC90aWZmOkNvbXByZXNzaW9uPgogICAgICAgICA8dGlmZjpQaG90b21ldHJpY0ludGVycHJldGF0aW9uPjI8L3RpZmY6UGhvdG9tZXRyaWNJbnRlcnByZXRhdGlvbj4KICAgICAgICAgPHRpZmY6T3JpZW50YXRpb24+MTwvdGlmZjpPcmllbnRhdGlvbj4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC94OnhtcG1ldGE+Cjl0tmoAAAHVSURBVFgJ7VZRsoMgDNTOu5E9U+/Ud6Z6JssGNg2oNKD90xkHCNnNkgTbYbieKwNXBn6bgSXQ4+16xi5UDiqDN3Pecr6+1fM5DHh7n1NEIPjjoRLKzOjG3qQ5dRtEy2LCjh/Gz2wDZE2nZYKkrxdn/kY9XQQkGCGqqDY5IgJFkEKgBCzDNGXhTKEye7boFRH6IPJj5EshiNCSjV4R4eSx7zhmR2tcdIuwmWiMeao7e0JHViZEWUI5aP8a9O+rx74D6sGEiJftiX3YeueIiFXg2KrhpqzjVC3dPZFYJZ7NOwwtNwM8R0UkLfH0sT5qck+OlkMq0BucKr0iWG7gpAQksD9esM1z3Lnf6SHjLh67nnKEGxC/iomWhByTeXOQJGHHcKxwHhHKnt1HIdYtmexkIb/HOURWTSJqn2gKMDG0bDUc/D0iAseovxUBoylmQCug6IVhSv+4DIeKI94jAr4AjiSEgQ25JYB+YWT9BZ94AM8erwgFkRifaArA6U0G5KT0m//z26REZuK9okgrT6VwE1jTHjbVzyNAyRwTEPOtuiex9FVBNZCkruaA4PZqFp1u8Rpww9/6rcK5y0EkAxRiZJt79PWOVYWGRE9pbJhavMengMflGyumk0akMsQnAAAAAElFTkSuQmCC' @@ -183,3 +188,58 @@ def test_ui_settings(get, put, patch, delete, admin, enterprise_license): response = get(url, user=admin, expect=200) assert not response.data['CUSTOM_LOGO'] assert not response.data['CUSTOM_LOGIN_INFO'] + + +@pytest.mark.django_db +def test_logging_aggregrator_connection_test_requires_superuser(get, post, alice): + url = reverse('api:setting_logging_test') + post(url, {}, user=alice, expect=403) + + +@pytest.mark.parametrize('key', [ + 'LOG_AGGREGATOR_TYPE', + 'LOG_AGGREGATOR_HOST', + 'LOG_AGGREGATOR_PORT', +]) +@pytest.mark.django_db +def test_logging_aggregrator_connection_test_bad_request(get, post, admin, key): + url = reverse('api:setting_logging_test') + resp = post(url, {}, user=admin, expect=400) + assert 'This field is required.' in resp.data.get(key, []) + + +@pytest.mark.django_db +def test_logging_aggregrator_connection_test_valid(mocker, get, post, admin): + with mock.patch.object(BaseHTTPSHandler, 'perform_test') as perform_test: + url = reverse('api:setting_logging_test') + post(url, { + 'LOG_AGGREGATOR_TYPE': 'logstash', + 'LOG_AGGREGATOR_HOST': 'localhost', + 'LOG_AGGREGATOR_PORT': 8080, + 'LOG_AGGREGATOR_USERNAME': 'logger', + 'LOG_AGGREGATOR_PASSWORD': 'mcstash' + }, user=admin, expect=200) + perform_test.assert_called_with(OrderedDict([ + ('LOG_AGGREGATOR_HOST', u'localhost'), + ('LOG_AGGREGATOR_PORT', 8080), + ('LOG_AGGREGATOR_TYPE', 'logstash'), + ('LOG_AGGREGATOR_USERNAME', 'logger'), + ('LOG_AGGREGATOR_PASSWORD', 'mcstash'), + ('LOG_AGGREGATOR_LOGGERS', ['awx', 'activity_stream', 'job_events', 'system_tracking']), + ('LOG_AGGREGATOR_INDIVIDUAL_FACTS', False), + ('LOG_AGGREGATOR_ENABLED', False), + ('LOG_AGGREGATOR_TOWER_UUID', '') + ])) + + +@pytest.mark.django_db +def test_logging_aggregrator_connection_test_invalid(mocker, get, post, admin): + with mock.patch.object(BaseHTTPSHandler, 'perform_test') as perform_test: + perform_test.side_effect = LoggingConnectivityException('404: Not Found') + url = reverse('api:setting_logging_test') + resp = post(url, { + 'LOG_AGGREGATOR_TYPE': 'logstash', + 'LOG_AGGREGATOR_HOST': 'localhost', + 'LOG_AGGREGATOR_PORT': 8080 + }, user=admin, expect=500) + assert resp.data == {'error': '404: Not Found'} diff --git a/awx/main/tests/unit/utils/test_handlers.py b/awx/main/tests/unit/utils/test_handlers.py index 3de3b2e7b7..0f4dbdff05 100644 --- a/awx/main/tests/unit/utils/test_handlers.py +++ b/awx/main/tests/unit/utils/test_handlers.py @@ -1,13 +1,15 @@ import base64 import json import logging +from uuid import uuid4 from django.conf import LazySettings import pytest import requests from requests_futures.sessions import FuturesSession -from awx.main.utils.handlers import BaseHTTPSHandler as HTTPSHandler, PARAM_NAMES +from awx.main.utils.handlers import (BaseHTTPSHandler as HTTPSHandler, + PARAM_NAMES, LoggingConnectivityException) from awx.main.utils.formatters import LogstashFormatter @@ -25,19 +27,21 @@ def dummy_log_record(): @pytest.fixture() -def ok200_adapter(): - class OK200Adapter(requests.adapters.HTTPAdapter): +def http_adapter(): + class FakeHTTPAdapter(requests.adapters.HTTPAdapter): requests = [] + status = 200 + reason = None def send(self, request, **kwargs): self.requests.append(request) resp = requests.models.Response() - resp.status_code = 200 - resp.raw = '200 OK' + resp.status_code = self.status + resp.reason = self.reason resp.request = request return resp - return OK200Adapter() + return FakeHTTPAdapter() def test_https_logging_handler_requests_sync_implementation(): @@ -73,6 +77,42 @@ def test_https_logging_handler_from_django_settings(param, django_settings_name) assert hasattr(handler, param) and getattr(handler, param) == 'EXAMPLE' +@pytest.mark.parametrize( + 'status, reason, exc', + [(200, '200 OK', None), (404, 'Not Found', LoggingConnectivityException)] +) +def test_https_logging_handler_connectivity_test(http_adapter, status, reason, exc): + http_adapter.status = status + http_adapter.reason = reason + settings = LazySettings() + settings.configure(**{ + 'LOG_AGGREGATOR_HOST': 'example.org', + 'LOG_AGGREGATOR_PORT': 8080, + 'LOG_AGGREGATOR_TYPE': 'logstash', + 'LOG_AGGREGATOR_USERNAME': 'user', + 'LOG_AGGREGATOR_PASSWORD': 'password', + 'LOG_AGGREGATOR_LOGGERS': ['awx', 'activity_stream', 'job_events', 'system_tracking'], + 'CLUSTER_HOST_ID': '', + 'LOG_AGGREGATOR_TOWER_UUID': str(uuid4()) + }) + + class FakeHTTPSHandler(HTTPSHandler): + + def __init__(self, *args, **kwargs): + super(FakeHTTPSHandler, self).__init__(*args, **kwargs) + self.session.mount('http://', http_adapter) + + def emit(self, record): + return super(FakeHTTPSHandler, self).emit(record) + + if exc: + with pytest.raises(exc) as e: + FakeHTTPSHandler.perform_test(settings) + assert str(e).endswith('%s: %s' % (status, reason)) + else: + assert FakeHTTPSHandler.perform_test(settings) is None + + def test_https_logging_handler_logstash_auth_info(): handler = HTTPSHandler(message_type='logstash', username='bob', password='ansible') handler.add_auth_information() @@ -120,19 +160,19 @@ def test_https_logging_handler_skip_log(params, logger_name, expected): ('splunk', False), ('splunk', True), ]) -def test_https_logging_handler_emit(ok200_adapter, dummy_log_record, +def test_https_logging_handler_emit(http_adapter, dummy_log_record, message_type, async): handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, message_type=message_type, enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'], async=async) handler.setFormatter(LogstashFormatter()) - handler.session.mount('http://', ok200_adapter) + handler.session.mount('http://', http_adapter) async_futures = handler.emit(dummy_log_record) [future.result() for future in async_futures] - assert len(ok200_adapter.requests) == 1 - request = ok200_adapter.requests[0] + assert len(http_adapter.requests) == 1 + request = http_adapter.requests[0] assert request.url == 'http://127.0.0.1/' assert request.method == 'POST' body = json.loads(request.body) @@ -152,7 +192,7 @@ def test_https_logging_handler_emit(ok200_adapter, dummy_log_record, @pytest.mark.parametrize('async', (True, False)) -def test_https_logging_handler_emit_logstash_with_creds(ok200_adapter, +def test_https_logging_handler_emit_logstash_with_creds(http_adapter, dummy_log_record, async): handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, username='user', password='pass', @@ -160,38 +200,38 @@ def test_https_logging_handler_emit_logstash_with_creds(ok200_adapter, enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'], async=async) handler.setFormatter(LogstashFormatter()) - handler.session.mount('http://', ok200_adapter) + handler.session.mount('http://', http_adapter) async_futures = handler.emit(dummy_log_record) [future.result() for future in async_futures] - assert len(ok200_adapter.requests) == 1 - request = ok200_adapter.requests[0] + assert len(http_adapter.requests) == 1 + request = http_adapter.requests[0] assert request.headers['Authorization'] == 'Basic %s' % base64.b64encode("user:pass") @pytest.mark.parametrize('async', (True, False)) -def test_https_logging_handler_emit_splunk_with_creds(ok200_adapter, +def test_https_logging_handler_emit_splunk_with_creds(http_adapter, dummy_log_record, async): handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, password='pass', message_type='splunk', enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'], async=async) handler.setFormatter(LogstashFormatter()) - handler.session.mount('http://', ok200_adapter) + handler.session.mount('http://', http_adapter) async_futures = handler.emit(dummy_log_record) [future.result() for future in async_futures] - assert len(ok200_adapter.requests) == 1 - request = ok200_adapter.requests[0] + assert len(http_adapter.requests) == 1 + request = http_adapter.requests[0] assert request.headers['Authorization'] == 'Splunk pass' -def test_https_logging_handler_emit_one_record_per_fact(ok200_adapter): +def test_https_logging_handler_emit_one_record_per_fact(http_adapter): handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, message_type='logstash', indv_facts=True, enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) handler.setFormatter(LogstashFormatter()) - handler.session.mount('http://', ok200_adapter) + handler.session.mount('http://', http_adapter) record = logging.LogRecord( 'awx.analytics.system_tracking', # logger name 20, # loglevel INFO @@ -212,8 +252,8 @@ def test_https_logging_handler_emit_one_record_per_fact(ok200_adapter): async_futures = handler.emit(record) [future.result() for future in async_futures] - assert len(ok200_adapter.requests) == 2 - requests = sorted(ok200_adapter.requests, key=lambda request: json.loads(request.body)['version']) + assert len(http_adapter.requests) == 2 + requests = sorted(http_adapter.requests, key=lambda request: json.loads(request.body)['version']) request = requests[0] assert request.url == 'http://127.0.0.1/' diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py index fe2fb87228..a258b8794c 100644 --- a/awx/main/utils/handlers.py +++ b/awx/main/utils/handlers.py @@ -5,6 +5,7 @@ import logging import json import requests +from requests.exceptions import RequestException from copy import copy # loggly @@ -40,6 +41,10 @@ def unused_callback(sess, resp): pass +class LoggingConnectivityException(Exception): + pass + + class HTTPSNullHandler(logging.NullHandler): "Placeholder null handler to allow loading without database access" @@ -66,6 +71,31 @@ class BaseHTTPSHandler(logging.Handler): kwargs[param] = getattr(settings, django_setting_name, None) return cls(*args, **kwargs) + @classmethod + def perform_test(cls, settings): + """ + Tests logging connectivity for the current logging settings. + @raises LoggingConnectivityException + """ + handler = cls.from_django_settings(settings, async=True) + handler.enabled_flag = True + handler.setFormatter(LogstashFormatter(settings_module=settings)) + logger = logging.getLogger(__file__) + fn, lno, func = logger.findCaller() + record = logger.makeRecord('awx', 10, fn, lno, + 'Ansible Tower Connection Test', tuple(), + None, func) + futures = handler.emit(record) + for future in futures: + try: + resp = future.result() + if not resp.ok: + raise LoggingConnectivityException( + ': '.join([str(resp.status_code), resp.reason or '']) + ) + except RequestException as e: + raise LoggingConnectivityException(str(e)) + def get_full_message(self, record): if record.exc_info: return '\n'.join(traceback.format_exception(*record.exc_info))