From ac20aa954ad9471d2b67a9c55c7e85f3f47c540a Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 30 Apr 2018 15:51:25 -0400 Subject: [PATCH] Replace logging-related restart with dynamic handler refactor existing handlers to be the related "real" handler classes, which are swapped out dynamically by external logger "proxy" handler class real handler swapout only done on setting change remove restart_local_services method get rid of uWSGI fifo file change TCP/UDP return type contract so that it mirrors the request futures object add details to socket error messages --- Makefile | 2 +- awx/conf/apps.py | 3 - awx/conf/tests/functional/test_api.py | 11 +- awx/conf/views.py | 7 +- awx/main/tasks.py | 18 +- .../tests/functional/api/test_settings.py | 14 +- awx/main/tests/unit/conftest.py | 14 + awx/main/tests/unit/utils/test_filters.py | 33 ++- awx/main/tests/unit/utils/test_handlers.py | 202 +++++-------- awx/main/tests/unit/utils/test_reload.py | 28 -- awx/main/utils/filters.py | 100 ++++++- awx/main/utils/formatters.py | 26 +- awx/main/utils/handlers.py | 279 +++++++++--------- awx/main/utils/reload.py | 35 --- awx/settings/defaults.py | 14 +- awx/settings/development.py | 2 - awx/settings/production.py | 2 - 17 files changed, 396 insertions(+), 394 deletions(-) diff --git a/Makefile b/Makefile index e7df5b3d48..f4e4984b24 100644 --- a/Makefile +++ b/Makefile @@ -296,7 +296,7 @@ uwsgi: collectstatic @if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/awx/bin/activate; \ fi; \ - uwsgi -b 32768 --socket 127.0.0.1:8050 --module=awx.wsgi:application --home=/venv/awx --chdir=/awx_devel/ --vacuum --processes=5 --harakiri=120 --master --no-orphans --py-autoreload 1 --max-requests=1000 --stats /tmp/stats.socket --master-fifo=/awxfifo --lazy-apps --logformat "%(addr) %(method) %(uri) - %(proto) %(status)" --hook-accepting1-once="exec:/bin/sh -c '[ -f /tmp/celery_pid ] && kill -1 `cat /tmp/celery_pid` || true'" + uwsgi -b 32768 --socket 127.0.0.1:8050 --module=awx.wsgi:application --home=/venv/awx --chdir=/awx_devel/ --vacuum --processes=5 --harakiri=120 --master --no-orphans --py-autoreload 1 --max-requests=1000 --stats /tmp/stats.socket --lazy-apps --logformat "%(addr) %(method) %(uri) - %(proto) %(status)" --hook-accepting1-once="exec:/bin/sh -c '[ -f /tmp/celery_pid ] && kill -1 `cat /tmp/celery_pid` || true'" daphne: @if [ "$(VENV_BASE)" ]; then \ diff --git a/awx/conf/apps.py b/awx/conf/apps.py index a70d21326c..4f9a36395c 100644 --- a/awx/conf/apps.py +++ b/awx/conf/apps.py @@ -2,8 +2,6 @@ from django.apps import AppConfig # from django.core import checks from django.utils.translation import ugettext_lazy as _ -from awx.main.utils.handlers import configure_external_logger -from django.conf import settings class ConfConfig(AppConfig): @@ -15,4 +13,3 @@ class ConfConfig(AppConfig): self.module.autodiscover() from .settings import SettingsWrapper SettingsWrapper.initialize() - configure_external_logger(settings) diff --git a/awx/conf/tests/functional/test_api.py b/awx/conf/tests/functional/test_api.py index 0e845238f6..1be22ee2dc 100644 --- a/awx/conf/tests/functional/test_api.py +++ b/awx/conf/tests/functional/test_api.py @@ -338,13 +338,14 @@ def test_setting_singleton_delete_no_read_only_fields(api_request, dummy_setting @pytest.mark.django_db def test_setting_logging_test(api_request): - with mock.patch('awx.conf.views.BaseHTTPSHandler.perform_test') as mock_func: + with mock.patch('awx.conf.views.AWXProxyHandler.perform_test') as mock_func: api_request( 'post', reverse('api:setting_logging_test'), data={'LOG_AGGREGATOR_HOST': 'http://foobar', 'LOG_AGGREGATOR_TYPE': 'logstash'} ) - test_arguments = mock_func.call_args[0][0] - assert test_arguments.LOG_AGGREGATOR_HOST == 'http://foobar' - assert test_arguments.LOG_AGGREGATOR_TYPE == 'logstash' - assert test_arguments.LOG_AGGREGATOR_LEVEL == 'DEBUG' + call = mock_func.call_args_list[0] + args, kwargs = call + given_settings = kwargs['custom_settings'] + assert given_settings.LOG_AGGREGATOR_HOST == 'http://foobar' + assert given_settings.LOG_AGGREGATOR_TYPE == 'logstash' diff --git a/awx/conf/views.py b/awx/conf/views.py index 60ea39d911..e10fe7ad32 100644 --- a/awx/conf/views.py +++ b/awx/conf/views.py @@ -21,7 +21,7 @@ from awx.api.generics import * # noqa from awx.api.permissions import IsSuperUser from awx.api.versioning import reverse, get_request_version from awx.main.utils import * # noqa -from awx.main.utils.handlers import BaseHTTPSHandler, UDPHandler, LoggingConnectivityException +from awx.main.utils.handlers import AWXProxyHandler, LoggingConnectivityException from awx.main.tasks import handle_setting_changes from awx.conf.license import get_licensed_features from awx.conf.models import Setting @@ -198,12 +198,9 @@ class SettingLoggingTest(GenericAPIView): mock_settings = MockSettings() for k, v in serializer.validated_data.items(): setattr(mock_settings, k, v) - mock_settings.LOG_AGGREGATOR_LEVEL = 'DEBUG' + AWXProxyHandler().perform_test(custom_settings=mock_settings) if mock_settings.LOG_AGGREGATOR_PROTOCOL.upper() == 'UDP': - UDPHandler.perform_test(mock_settings) return Response(status=status.HTTP_201_CREATED) - else: - BaseHTTPSHandler.perform_test(mock_settings) except LoggingConnectivityException as e: return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) return Response(status=status.HTTP_200_OK) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index cdf2a11354..22e6dff090 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -29,7 +29,7 @@ except Exception: # Celery from celery import Task, shared_task, Celery -from celery.signals import celeryd_init, worker_process_init, worker_shutdown, worker_ready, celeryd_after_setup +from celery.signals import celeryd_init, worker_shutdown, worker_ready, celeryd_after_setup # Django from django.conf import settings @@ -59,10 +59,9 @@ from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, wrap_args_with_proot, OutputEventFilter, OutputVerboseFilter, ignore_inventory_computed_fields, ignore_inventory_group_removal, get_type_for_model, extract_ansible_vars) from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja -from awx.main.utils.reload import restart_local_services, stop_local_services +from awx.main.utils.reload import stop_local_services from awx.main.utils.pglock import advisory_lock from awx.main.utils.ha import update_celery_worker_routes, register_celery_worker_queues -from awx.main.utils.handlers import configure_external_logger from awx.main.consumers import emit_channel_notification from awx.conf import settings_registry @@ -117,15 +116,6 @@ def celery_startup(conf=None, **kwargs): logger.exception(six.text_type("Failed to rebuild schedule {}.").format(sch)) -@worker_process_init.connect -def task_set_logger_pre_run(*args, **kwargs): - try: - cache.close() - configure_external_logger(settings, is_startup=False) - except Exception: - logger.exception('Encountered error on initial log configuration.') - - @worker_shutdown.connect def inform_cluster_of_shutdown(*args, **kwargs): try: @@ -200,10 +190,6 @@ def handle_setting_changes(self, setting_keys): cache_keys = set(setting_keys) logger.debug('cache delete_many(%r)', cache_keys) cache.delete_many(cache_keys) - for key in cache_keys: - if key.startswith('LOG_AGGREGATOR_'): - restart_local_services(['uwsgi', 'celery', 'beat', 'callback']) - break @shared_task(bind=True, exchange='tower_broadcast_all') diff --git a/awx/main/tests/functional/api/test_settings.py b/awx/main/tests/functional/api/test_settings.py index 5974a1a935..97effe0fa3 100644 --- a/awx/main/tests/functional/api/test_settings.py +++ b/awx/main/tests/functional/api/test_settings.py @@ -14,7 +14,7 @@ import mock # AWX from awx.api.versioning import reverse from awx.conf.models import Setting -from awx.main.utils.handlers import BaseHTTPSHandler, LoggingConnectivityException +from awx.main.utils.handlers import AWXProxyHandler, LoggingConnectivityException import six @@ -217,7 +217,7 @@ def test_logging_aggregrator_connection_test_bad_request(get, post, admin, key): @pytest.mark.django_db def test_logging_aggregrator_connection_test_valid(mocker, get, post, admin): - with mock.patch.object(BaseHTTPSHandler, 'perform_test') as perform_test: + with mock.patch.object(AWXProxyHandler, 'perform_test') as perform_test: url = reverse('api:setting_logging_test') user_data = { 'LOG_AGGREGATOR_TYPE': 'logstash', @@ -227,7 +227,8 @@ def test_logging_aggregrator_connection_test_valid(mocker, get, post, admin): 'LOG_AGGREGATOR_PASSWORD': 'mcstash' } post(url, user_data, user=admin, expect=200) - create_settings = perform_test.call_args[0][0] + args, kwargs = perform_test.call_args_list[0] + create_settings = kwargs['custom_settings'] for k, v in user_data.items(): assert hasattr(create_settings, k) assert getattr(create_settings, k) == v @@ -238,7 +239,7 @@ def test_logging_aggregrator_connection_test_with_masked_password(mocker, patch, url = reverse('api:setting_singleton_detail', kwargs={'category_slug': 'logging'}) patch(url, user=admin, data={'LOG_AGGREGATOR_PASSWORD': 'password123'}, expect=200) - with mock.patch.object(BaseHTTPSHandler, 'perform_test') as perform_test: + with mock.patch.object(AWXProxyHandler, 'perform_test') as perform_test: url = reverse('api:setting_logging_test') user_data = { 'LOG_AGGREGATOR_TYPE': 'logstash', @@ -248,13 +249,14 @@ def test_logging_aggregrator_connection_test_with_masked_password(mocker, patch, 'LOG_AGGREGATOR_PASSWORD': '$encrypted$' } post(url, user_data, user=admin, expect=200) - create_settings = perform_test.call_args[0][0] + args, kwargs = perform_test.call_args_list[0] + create_settings = kwargs['custom_settings'] assert getattr(create_settings, 'LOG_AGGREGATOR_PASSWORD') == 'password123' @pytest.mark.django_db def test_logging_aggregrator_connection_test_invalid(mocker, get, post, admin): - with mock.patch.object(BaseHTTPSHandler, 'perform_test') as perform_test: + with mock.patch.object(AWXProxyHandler, 'perform_test') as perform_test: perform_test.side_effect = LoggingConnectivityException('404: Not Found') url = reverse('api:setting_logging_test') resp = post(url, { diff --git a/awx/main/tests/unit/conftest.py b/awx/main/tests/unit/conftest.py index 7a5ae40f79..2307b3a47d 100644 --- a/awx/main/tests/unit/conftest.py +++ b/awx/main/tests/unit/conftest.py @@ -1,4 +1,5 @@ import pytest +import logging from mock import PropertyMock @@ -7,3 +8,16 @@ from mock import PropertyMock def _disable_database_settings(mocker): m = mocker.patch('awx.conf.settings.SettingsWrapper.all_supported_settings', new_callable=PropertyMock) m.return_value = [] + + +@pytest.fixture() +def dummy_log_record(): + return logging.LogRecord( + 'awx', # logger name + 20, # loglevel INFO + './awx/some/module.py', # pathname + 100, # lineno + 'User joe logged in', # msg + tuple(), # args, + None # exc_info + ) diff --git a/awx/main/tests/unit/utils/test_filters.py b/awx/main/tests/unit/utils/test_filters.py index c0b38d294c..a8127fdfb6 100644 --- a/awx/main/tests/unit/utils/test_filters.py +++ b/awx/main/tests/unit/utils/test_filters.py @@ -5,7 +5,7 @@ import mock from collections import namedtuple # AWX -from awx.main.utils.filters import SmartFilter +from awx.main.utils.filters import SmartFilter, ExternalLoggerEnabled # Django from django.db.models import Q @@ -13,6 +13,37 @@ from django.db.models import Q import six +@pytest.mark.parametrize('params, logger_name, expected', [ + # skip all records if enabled_flag = False + ({'enabled_flag': False}, 'awx.main', False), + # skip all records if the host is undefined + ({'enabled_flag': True}, 'awx.main', False), + # skip all records if underlying logger is used by handlers themselves + ({'enabled_flag': True}, 'awx.main.utils.handlers', False), + ({'enabled_flag': True, 'enabled_loggers': ['awx']}, 'awx.main', True), + ({'enabled_flag': True, 'enabled_loggers': ['abc']}, 'awx.analytics.xyz', False), + ({'enabled_flag': True, 'enabled_loggers': ['xyz']}, 'awx.analytics.xyz', True), +]) +def test_base_logging_handler_skip_log(params, logger_name, expected, dummy_log_record): + filter = ExternalLoggerEnabled(**params) + dummy_log_record.name = logger_name + assert filter.filter(dummy_log_record) is expected, (params, logger_name) + + +@pytest.mark.parametrize('level, expect', [ + (30, True), # warning + (20, False) # info +]) +def test_log_configurable_severity(level, expect, dummy_log_record): + dummy_log_record.levelno = level + filter = ExternalLoggerEnabled( + enabled_flag=True, + enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'], + lvl='WARNING' + ) + assert filter.filter(dummy_log_record) is expect + + Field = namedtuple('Field', 'name') Meta = namedtuple('Meta', 'fields') diff --git a/awx/main/tests/unit/utils/test_handlers.py b/awx/main/tests/unit/utils/test_handlers.py index 693d024281..f57d86158d 100644 --- a/awx/main/tests/unit/utils/test_handlers.py +++ b/awx/main/tests/unit/utils/test_handlers.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import base64 import cStringIO -import json import logging import socket import datetime @@ -10,7 +9,6 @@ from uuid import uuid4 import mock -from django.conf import settings from django.conf import LazySettings import pytest import requests @@ -18,23 +16,11 @@ from requests_futures.sessions import FuturesSession from awx.main.utils.handlers import (BaseHandler, BaseHTTPSHandler as HTTPSHandler, TCPHandler, UDPHandler, _encode_payload_for_socket, - PARAM_NAMES, LoggingConnectivityException) + PARAM_NAMES, LoggingConnectivityException, + AWXProxyHandler) from awx.main.utils.formatters import LogstashFormatter -@pytest.fixture() -def dummy_log_record(): - return logging.LogRecord( - 'awx', # logger name - 20, # loglevel INFO - './awx/some/module.py', # pathname - 100, # lineno - 'User joe logged in', # msg - tuple(), # args, - None # exc_info - ) - - @pytest.fixture() def http_adapter(): class FakeHTTPAdapter(requests.adapters.HTTPAdapter): @@ -80,105 +66,91 @@ def test_https_logging_handler_requests_async_implementation(): def test_https_logging_handler_has_default_http_timeout(): - handler = HTTPSHandler.from_django_settings(settings) + handler = TCPHandler() assert handler.tcp_timeout == 5 -@pytest.mark.parametrize('param', PARAM_NAMES.keys()) +@pytest.mark.parametrize('param', ['host', 'port', 'indv_facts']) def test_base_logging_handler_defaults(param): handler = BaseHandler() assert hasattr(handler, param) and getattr(handler, param) is None -@pytest.mark.parametrize('param', PARAM_NAMES.keys()) +@pytest.mark.parametrize('param', ['host', 'port', 'indv_facts']) def test_base_logging_handler_kwargs(param): handler = BaseHandler(**{param: 'EXAMPLE'}) assert hasattr(handler, param) and getattr(handler, param) == 'EXAMPLE' -@pytest.mark.parametrize('param, django_settings_name', PARAM_NAMES.items()) -def test_base_logging_handler_from_django_settings(param, django_settings_name): +@pytest.mark.parametrize('params', [ + { + 'LOG_AGGREGATOR_HOST': 'https://server.invalid', + 'LOG_AGGREGATOR_PORT': 22222, + 'LOG_AGGREGATOR_TYPE': 'loggly', + 'LOG_AGGREGATOR_USERNAME': 'foo', + 'LOG_AGGREGATOR_PASSWORD': 'bar', + 'LOG_AGGREGATOR_INDIVIDUAL_FACTS': True, + 'LOG_AGGREGATOR_TCP_TIMEOUT': 96, + 'LOG_AGGREGATOR_VERIFY_CERT': False, + 'LOG_AGGREGATOR_PROTOCOL': 'https' + }, + { + 'LOG_AGGREGATOR_HOST': 'https://server.invalid', + 'LOG_AGGREGATOR_PORT': 22222, + 'LOG_AGGREGATOR_PROTOCOL': 'udp' + } +]) +def test_real_handler_from_django_settings(params): + settings = LazySettings() + settings.configure(**params) + handler = AWXProxyHandler().get_handler(custom_settings=settings) + # need the _reverse_ dictionary from PARAM_NAMES + attr_lookup = {} + for attr_name, setting_name in PARAM_NAMES.items(): + attr_lookup[setting_name] = attr_name + for setting_name, val in params.items(): + attr_name = attr_lookup[setting_name] + if attr_name == 'protocol': + continue + assert hasattr(handler, attr_name) + + +def test_invalid_kwarg_to_real_handler(): settings = LazySettings() settings.configure(**{ - django_settings_name: 'EXAMPLE' + 'LOG_AGGREGATOR_HOST': 'https://server.invalid', + 'LOG_AGGREGATOR_PORT': 22222, + 'LOG_AGGREGATOR_PROTOCOL': 'udp', + 'LOG_AGGREGATOR_VERIFY_CERT': False # setting not valid for UDP handler }) - handler = BaseHandler.from_django_settings(settings) - assert hasattr(handler, param) and getattr(handler, param) == 'EXAMPLE' + handler = AWXProxyHandler().get_handler(custom_settings=settings) + assert not hasattr(handler, 'verify_cert') -@pytest.mark.parametrize('params, logger_name, expected', [ - # skip all records if enabled_flag = False - ({'enabled_flag': False}, 'awx.main', True), - # skip all records if the host is undefined - ({'host': '', 'enabled_flag': True}, 'awx.main', True), - # skip all records if underlying logger is used by handlers themselves - ({'host': '127.0.0.1', 'enabled_flag': True}, 'awx.main.utils.handlers', True), - ({'host': '127.0.0.1', 'enabled_flag': True}, 'awx.main', False), - ({'host': '127.0.0.1', 'enabled_flag': True, 'enabled_loggers': ['abc']}, 'awx.analytics.xyz', True), - ({'host': '127.0.0.1', 'enabled_flag': True, 'enabled_loggers': ['xyz']}, 'awx.analytics.xyz', False), -]) -def test_base_logging_handler_skip_log(params, logger_name, expected): - handler = BaseHandler(**params) - assert handler._skip_log(logger_name) is expected - - -def test_base_logging_handler_emit(dummy_log_record): - handler = BaseHandler(host='127.0.0.1', enabled_flag=True, - message_type='logstash', lvl='INFO', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) +def test_base_logging_handler_emit_system_tracking(dummy_log_record): + handler = BaseHandler(host='127.0.0.1', indv_facts=True) handler.setFormatter(LogstashFormatter()) - sent_payloads = handler.emit(dummy_log_record) - - assert len(sent_payloads) == 1 - body = json.loads(sent_payloads[0]) - - assert body['level'] == 'INFO' - assert body['logger_name'] == 'awx' - assert body['message'] == 'User joe logged in' - - -def test_base_logging_handler_ignore_low_severity_msg(dummy_log_record): - handler = BaseHandler(host='127.0.0.1', enabled_flag=True, - message_type='logstash', lvl='WARNING', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) - handler.setFormatter(LogstashFormatter()) - sent_payloads = handler.emit(dummy_log_record) - assert len(sent_payloads) == 0 - - -def test_base_logging_handler_emit_system_tracking(): - handler = BaseHandler(host='127.0.0.1', enabled_flag=True, - message_type='logstash', indv_facts=True, lvl='INFO', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) - handler.setFormatter(LogstashFormatter()) - record = logging.LogRecord( - 'awx.analytics.system_tracking', # logger name - 20, # loglevel INFO - './awx/some/module.py', # pathname - 100, # lineno - None, # msg - tuple(), # args, - None # exc_info - ) - record.inventory_id = 11 - record.host_name = 'my_lucky_host' - record.job_id = 777 - record.ansible_facts = { + dummy_log_record.name = 'awx.analytics.system_tracking' + dummy_log_record.msg = None + dummy_log_record.inventory_id = 11 + dummy_log_record.host_name = 'my_lucky_host' + dummy_log_record.job_id = 777 + dummy_log_record.ansible_facts = { "ansible_kernel": "4.4.66-boot2docker", "ansible_machine": "x86_64", "ansible_swapfree_mb": 4663, } - record.ansible_facts_modified = datetime.datetime.now(tzutc()).isoformat() - sent_payloads = handler.emit(record) + dummy_log_record.ansible_facts_modified = datetime.datetime.now(tzutc()).isoformat() + sent_payloads = handler.emit(dummy_log_record) assert len(sent_payloads) == 1 - assert sent_payloads[0]['ansible_facts'] == record.ansible_facts - assert sent_payloads[0]['ansible_facts_modified'] == record.ansible_facts_modified + assert sent_payloads[0]['ansible_facts'] == dummy_log_record.ansible_facts + assert sent_payloads[0]['ansible_facts_modified'] == dummy_log_record.ansible_facts_modified assert sent_payloads[0]['level'] == 'INFO' assert sent_payloads[0]['logger_name'] == 'awx.analytics.system_tracking' - assert sent_payloads[0]['job_id'] == record.job_id - assert sent_payloads[0]['inventory_id'] == record.inventory_id - assert sent_payloads[0]['host_name'] == record.host_name + assert sent_payloads[0]['job_id'] == dummy_log_record.job_id + assert sent_payloads[0]['inventory_id'] == dummy_log_record.inventory_id + assert sent_payloads[0]['host_name'] == dummy_log_record.host_name @pytest.mark.parametrize('host, port, normalized, hostname_only', [ @@ -236,16 +208,18 @@ def test_https_logging_handler_connectivity_test(http_adapter, status, reason, e def emit(self, record): return super(FakeHTTPSHandler, self).emit(record) - if exc: - with pytest.raises(exc) as e: - FakeHTTPSHandler.perform_test(settings) - assert str(e).endswith('%s: %s' % (status, reason)) - else: - assert FakeHTTPSHandler.perform_test(settings) is None + with mock.patch.object(AWXProxyHandler, 'get_handler_class') as mock_get_class: + mock_get_class.return_value = FakeHTTPSHandler + if exc: + with pytest.raises(exc) as e: + AWXProxyHandler().perform_test(settings) + assert str(e).endswith('%s: %s' % (status, reason)) + else: + assert AWXProxyHandler().perform_test(settings) is None def test_https_logging_handler_logstash_auth_info(): - handler = HTTPSHandler(message_type='logstash', username='bob', password='ansible', lvl='INFO') + handler = HTTPSHandler(message_type='logstash', username='bob', password='ansible') handler._add_auth_information() assert isinstance(handler.session.auth, requests.auth.HTTPBasicAuth) assert handler.session.auth.username == 'bob' @@ -261,9 +235,7 @@ def test_https_logging_handler_splunk_auth_info(): def test_https_logging_handler_connection_error(connection_error_adapter, dummy_log_record): - handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, - message_type='logstash', lvl='INFO', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) + handler = HTTPSHandler(host='127.0.0.1', message_type='logstash') handler.setFormatter(LogstashFormatter()) handler.session.mount('http://', connection_error_adapter) @@ -289,9 +261,7 @@ def test_https_logging_handler_connection_error(connection_error_adapter, @pytest.mark.parametrize('message_type', ['logstash', 'splunk']) def test_https_logging_handler_emit_without_cred(http_adapter, dummy_log_record, message_type): - handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, - message_type=message_type, lvl='INFO', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) + handler = HTTPSHandler(host='127.0.0.1', message_type=message_type) handler.setFormatter(LogstashFormatter()) handler.session.mount('http://', http_adapter) async_futures = handler.emit(dummy_log_record) @@ -312,10 +282,9 @@ def test_https_logging_handler_emit_without_cred(http_adapter, dummy_log_record, def test_https_logging_handler_emit_logstash_with_creds(http_adapter, dummy_log_record): - handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, + handler = HTTPSHandler(host='127.0.0.1', username='user', password='pass', - message_type='logstash', lvl='INFO', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) + message_type='logstash') handler.setFormatter(LogstashFormatter()) handler.session.mount('http://', http_adapter) async_futures = handler.emit(dummy_log_record) @@ -328,9 +297,8 @@ def test_https_logging_handler_emit_logstash_with_creds(http_adapter, def test_https_logging_handler_emit_splunk_with_creds(http_adapter, dummy_log_record): - handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, - password='pass', message_type='splunk', lvl='INFO', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) + handler = HTTPSHandler(host='127.0.0.1', + password='pass', message_type='splunk') handler.setFormatter(LogstashFormatter()) handler.session.mount('http://', http_adapter) async_futures = handler.emit(dummy_log_record) @@ -351,9 +319,7 @@ def test_encode_payload_for_socket(payload, encoded_payload): def test_udp_handler_create_socket_at_init(): - handler = UDPHandler(host='127.0.0.1', port=4399, - enabled_flag=True, message_type='splunk', lvl='INFO', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) + handler = UDPHandler(host='127.0.0.1', port=4399) assert hasattr(handler, 'socket') assert isinstance(handler.socket, socket.socket) assert handler.socket.family == socket.AF_INET @@ -361,9 +327,7 @@ def test_udp_handler_create_socket_at_init(): def test_udp_handler_send(dummy_log_record): - handler = UDPHandler(host='127.0.0.1', port=4399, - enabled_flag=True, message_type='splunk', lvl='INFO', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) + handler = UDPHandler(host='127.0.0.1', port=4399) handler.setFormatter(LogstashFormatter()) with mock.patch('awx.main.utils.handlers._encode_payload_for_socket', return_value="des") as encode_mock,\ mock.patch.object(handler, 'socket') as socket_mock: @@ -373,9 +337,7 @@ def test_udp_handler_send(dummy_log_record): def test_tcp_handler_send(fake_socket, dummy_log_record): - handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5, - enabled_flag=True, message_type='splunk', lvl='INFO', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) + handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5) handler.setFormatter(LogstashFormatter()) with mock.patch('socket.socket', return_value=fake_socket) as sok_init_mock,\ mock.patch('select.select', return_value=([], [fake_socket], [])): @@ -388,9 +350,7 @@ def test_tcp_handler_send(fake_socket, dummy_log_record): def test_tcp_handler_return_if_socket_unavailable(fake_socket, dummy_log_record): - handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5, - enabled_flag=True, message_type='splunk', lvl='INFO', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) + handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5) handler.setFormatter(LogstashFormatter()) with mock.patch('socket.socket', return_value=fake_socket) as sok_init_mock,\ mock.patch('select.select', return_value=([], [], [])): @@ -403,9 +363,7 @@ def test_tcp_handler_return_if_socket_unavailable(fake_socket, dummy_log_record) def test_tcp_handler_log_exception(fake_socket, dummy_log_record): - handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5, - enabled_flag=True, message_type='splunk', lvl='INFO', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) + handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5) handler.setFormatter(LogstashFormatter()) with mock.patch('socket.socket', return_value=fake_socket) as sok_init_mock,\ mock.patch('select.select', return_value=([], [], [])),\ diff --git a/awx/main/tests/unit/utils/test_reload.py b/awx/main/tests/unit/utils/test_reload.py index 6d09d6105b..1820f2724a 100644 --- a/awx/main/tests/unit/utils/test_reload.py +++ b/awx/main/tests/unit/utils/test_reload.py @@ -13,31 +13,3 @@ def test_produce_supervisor_command(mocker): ['supervisorctl', 'restart', 'tower-processes:receiver',], stderr=-1, stdin=-1, stdout=-1) - -def test_routing_of_service_restarts_works(mocker): - ''' - This tests that the parent restart method will call the appropriate - service restart methods, depending on which services are given in args - ''' - with mocker.patch.object(reload, '_uwsgi_fifo_command'),\ - mocker.patch.object(reload, '_reset_celery_thread_pool'),\ - mocker.patch.object(reload, '_supervisor_service_command'): - reload.restart_local_services(['uwsgi', 'celery', 'flower', 'daphne']) - reload._uwsgi_fifo_command.assert_called_once_with(uwsgi_command="c") - reload._reset_celery_thread_pool.assert_called_once_with() - reload._supervisor_service_command.assert_called_once_with(['flower', 'daphne'], command="restart") - - - -def test_routing_of_service_restarts_diables(mocker): - ''' - Test that methods are not called if not in the args - ''' - with mocker.patch.object(reload, '_uwsgi_fifo_command'),\ - mocker.patch.object(reload, '_reset_celery_thread_pool'),\ - mocker.patch.object(reload, '_supervisor_service_command'): - reload.restart_local_services(['flower']) - reload._uwsgi_fifo_command.assert_not_called() - reload._reset_celery_thread_pool.assert_not_called() - reload._supervisor_service_command.assert_called_once_with(['flower'], command="restart") - diff --git a/awx/main/utils/filters.py b/awx/main/utils/filters.py index 81f91a0b0a..9563eb6c34 100644 --- a/awx/main/utils/filters.py +++ b/awx/main/utils/filters.py @@ -8,14 +8,106 @@ from pyparsing import ( CharsNotIn, ParseException, ) +from logging import Filter, _levelNames import six -import django +from django.apps import apps +from django.db import models +from django.conf import settings from awx.main.utils.common import get_search_fields -__all__ = ['SmartFilter'] +__all__ = ['SmartFilter', 'ExternalLoggerEnabled'] + + +class FieldFromSettings(object): + """ + Field interface - defaults to getting value from setting + if otherwise set, provided value will take precedence + over value in settings + """ + + def __init__(self, setting_name): + self.setting_name = setting_name + + def __get__(self, instance, type=None): + if self.setting_name in getattr(instance, 'settings_override', {}): + return instance.settings_override[self.setting_name] + return getattr(settings, self.setting_name, None) + + def __set__(self, instance, value): + if value is None: + if hasattr(instance, 'settings_override'): + instance.settings_override.pop('instance', None) + else: + if not hasattr(instance, 'settings_override'): + instance.settings_override = {} + instance.settings_override[self.setting_name] = value + + +class ExternalLoggerEnabled(Filter): + + # Prevents recursive logging loops from swamping the server + LOGGER_BLACKLIST = ( + # loggers that may be called in process of emitting a log + 'awx.main.utils.handlers', + 'awx.main.utils.formatters', + 'awx.main.utils.filters', + 'awx.main.utils.encryption', + 'awx.main.utils.log', + # loggers that may be called getting logging settings + 'awx.conf' + ) + + lvl = FieldFromSettings('LOG_AGGREGATOR_LEVEL') + enabled_loggers = FieldFromSettings('LOG_AGGREGATOR_LOGGERS') + enabled_flag = FieldFromSettings('LOG_AGGREGATOR_ENABLED') + + def __init__(self, **kwargs): + super(ExternalLoggerEnabled, self).__init__() + for field_name, field_value in kwargs.items(): + if not isinstance(ExternalLoggerEnabled.__dict__.get(field_name, None), FieldFromSettings): + raise Exception('%s is not a valid kwarg' % field_name) + if field_value is None: + continue + setattr(self, field_name, field_value) + + def filter(self, record): + """ + Uses the database settings to determine if the current + external log configuration says that this particular record + should be sent to the external log aggregator + + False - should not be logged + True - should be logged + """ + # Logger exceptions + for logger_name in self.LOGGER_BLACKLIST: + if record.name.startswith(logger_name): + return False + # General enablement + if not self.enabled_flag: + return False + + # Level enablement + if record.levelno < _levelNames[self.lvl]: + # logging._levelNames -> logging._nameToLevel in python 3 + return False + + # Logger type enablement + loggers = self.enabled_loggers + if not loggers: + return False + if record.name.startswith('awx.analytics'): + base_path, headline_name = record.name.rsplit('.', 1) + return bool(headline_name in loggers) + else: + if '.' in record.name: + base_name, trailing_path = record.name.split('.', 1) + else: + base_name = record.name + return bool(base_name in loggers) def string_to_type(t): @@ -36,7 +128,7 @@ def string_to_type(t): def get_model(name): - return django.apps.apps.get_model('main', name) + return apps.get_model('main', name) class SmartFilter(object): @@ -52,7 +144,7 @@ class SmartFilter(object): search_kwargs = self._expand_search(k, v) if search_kwargs: kwargs.update(search_kwargs) - q = reduce(lambda x, y: x | y, [django.db.models.Q(**{u'%s__contains' % _k:_v}) for _k, _v in kwargs.items()]) + q = reduce(lambda x, y: x | y, [models.Q(**{u'%s__contains' % _k:_v}) for _k, _v in kwargs.items()]) self.result = Host.objects.filter(q) else: kwargs[k] = v diff --git a/awx/main/utils/formatters.py b/awx/main/utils/formatters.py index c867912be4..f83dc3887c 100644 --- a/awx/main/utils/formatters.py +++ b/awx/main/utils/formatters.py @@ -9,6 +9,8 @@ import logging import six +from django.conf import settings + class TimeFormatter(logging.Formatter): ''' @@ -20,15 +22,6 @@ class TimeFormatter(logging.Formatter): class LogstashFormatter(LogstashFormatterVersion1): - def __init__(self, **kwargs): - settings_module = kwargs.pop('settings_module', None) - ret = super(LogstashFormatter, self).__init__(**kwargs) - if settings_module: - self.host_id = getattr(settings_module, 'CLUSTER_HOST_ID', None) - if hasattr(settings_module, 'LOG_AGGREGATOR_TOWER_UUID'): - self.tower_uuid = settings_module.LOG_AGGREGATOR_TOWER_UUID - self.message_type = getattr(settings_module, 'LOG_AGGREGATOR_TYPE', 'other') - return ret def reformat_data_for_log(self, raw_data, kind=None): ''' @@ -147,6 +140,15 @@ class LogstashFormatter(LogstashFormatterVersion1): if record.name.startswith('awx.analytics'): log_kind = record.name[len('awx.analytics.'):] fields = self.reformat_data_for_log(fields, kind=log_kind) + # General AWX metadata + for log_name, setting_name in [ + ('type', 'LOG_AGGREGATOR_TYPE'), + ('cluster_host_id', 'CLUSTER_HOST_ID'), + ('tower_uuid', 'LOG_AGGREGATOR_TOWER_UUID')]: + if hasattr(settings, setting_name): + fields[log_name] = getattr(settings, setting_name, None) + elif log_name == 'type': + fields[log_name] = 'other' return fields def format(self, record): @@ -158,18 +160,12 @@ class LogstashFormatter(LogstashFormatterVersion1): '@timestamp': self.format_timestamp(record.created), 'message': record.getMessage(), 'host': self.host, - 'type': self.message_type, # Extra Fields 'level': record.levelname, 'logger_name': record.name, } - if getattr(self, 'tower_uuid', None): - message['tower_uuid'] = self.tower_uuid - if getattr(self, 'host_id', None): - message['cluster_host_id'] = self.host_id - # Add extra fields message.update(self.get_extra_fields(record)) diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py index 8ed1127292..214c40ff11 100644 --- a/awx/main/utils/handlers.py +++ b/awx/main/utils/handlers.py @@ -13,40 +13,35 @@ import six from concurrent.futures import ThreadPoolExecutor from requests.exceptions import RequestException -# loggly -import traceback - +# Django from django.conf import settings + +# requests futures, a dependency used by these handlers from requests_futures.sessions import FuturesSession # AWX from awx.main.utils.formatters import LogstashFormatter -__all__ = ['HTTPSNullHandler', 'BaseHTTPSHandler', 'TCPHandler', 'UDPHandler', - 'configure_external_logger'] +__all__ = ['BaseHTTPSHandler', 'TCPHandler', 'UDPHandler', + 'AWXProxyHandler'] logger = logging.getLogger('awx.main.utils.handlers') -# AWX external logging handler, generally designed to be used -# with the accompanying LogstashHandler, derives from python-logstash library -# Non-blocking request accomplished by FuturesSession, similar -# to the loggly-python-handler library (not used) # Translation of parameter names to names in Django settings +# logging settings category, only those related to handler / log emission PARAM_NAMES = { 'host': 'LOG_AGGREGATOR_HOST', 'port': 'LOG_AGGREGATOR_PORT', 'message_type': 'LOG_AGGREGATOR_TYPE', 'username': 'LOG_AGGREGATOR_USERNAME', 'password': 'LOG_AGGREGATOR_PASSWORD', - 'enabled_loggers': 'LOG_AGGREGATOR_LOGGERS', 'indv_facts': 'LOG_AGGREGATOR_INDIVIDUAL_FACTS', - 'enabled_flag': 'LOG_AGGREGATOR_ENABLED', 'tcp_timeout': 'LOG_AGGREGATOR_TCP_TIMEOUT', 'verify_cert': 'LOG_AGGREGATOR_VERIFY_CERT', - 'lvl': 'LOG_AGGREGATOR_LEVEL', + 'protocol': 'LOG_AGGREGATOR_PROTOCOL' } @@ -58,13 +53,6 @@ class LoggingConnectivityException(Exception): pass -class HTTPSNullHandler(logging.NullHandler): - "Placeholder null handler to allow loading without database access" - - def __init__(self, *args, **kwargs): - return super(HTTPSNullHandler, self).__init__() - - class VerboseThreadPoolExecutor(ThreadPoolExecutor): last_log_emit = 0 @@ -91,32 +79,25 @@ class VerboseThreadPoolExecutor(ThreadPoolExecutor): **kwargs) -LEVEL_MAPPING = { - 'DEBUG': logging.DEBUG, - 'INFO': logging.INFO, - 'WARNING': logging.WARNING, - 'ERROR': logging.ERROR, - 'CRITICAL': logging.CRITICAL, -} +class SocketResult: + ''' + A class to be the return type of methods that send data over a socket + allows object to be used in the same way as a request futures object + ''' + def __init__(self, ok, reason=None): + self.ok = ok + self.reason = reason + + def result(self): + return self class BaseHandler(logging.Handler): - def __init__(self, **kwargs): + def __init__(self, host=None, port=None, indv_facts=None, **kwargs): super(BaseHandler, self).__init__() - for fd in PARAM_NAMES: - setattr(self, fd, kwargs.get(fd, None)) - - @classmethod - def from_django_settings(cls, settings, *args, **kwargs): - for param, django_setting_name in PARAM_NAMES.items(): - kwargs[param] = getattr(settings, django_setting_name, None) - return cls(*args, **kwargs) - - def get_full_message(self, record): - if record.exc_info: - return '\n'.join(traceback.format_exception(*record.exc_info)) - else: - return record.getMessage() + self.host = host + self.port = port + self.indv_facts = indv_facts def _send(self, payload): """Actually send message to log aggregator. @@ -128,26 +109,11 @@ class BaseHandler(logging.Handler): return [self._send(json.loads(self.format(record)))] return [self._send(self.format(record))] - def _skip_log(self, logger_name): - if self.host == '' or (not self.enabled_flag): - return True - # Don't send handler-related records. - if logger_name == logger.name: - return True - # AWX log emission is only turned off by enablement setting - if not logger_name.startswith('awx.analytics'): - return False - return self.enabled_loggers is None or logger_name[len('awx.analytics.'):] not in self.enabled_loggers - def emit(self, record): """ Emit a log record. Returns a list of zero or more implementation-specific objects for tests. """ - if not record.name.startswith('awx.analytics') and record.levelno < LEVEL_MAPPING[self.lvl]: - return [] - if self._skip_log(record.name): - return [] try: return self._format_and_send_record(record) except (KeyboardInterrupt, SystemExit): @@ -181,6 +147,11 @@ class BaseHandler(logging.Handler): class BaseHTTPSHandler(BaseHandler): + ''' + Originally derived from python-logstash library + Non-blocking request accomplished by FuturesSession, similar + to the loggly-python-handler library + ''' def _add_auth_information(self): if self.message_type == 'logstash': if not self.username: @@ -196,39 +167,20 @@ class BaseHTTPSHandler(BaseHandler): } self.session.headers.update(headers) - def __init__(self, fqdn=False, **kwargs): + def __init__(self, fqdn=False, message_type=None, username=None, password=None, + tcp_timeout=5, verify_cert=True, **kwargs): self.fqdn = fqdn + self.message_type = message_type + self.username = username + self.password = password + self.tcp_timeout = tcp_timeout + self.verify_cert = verify_cert super(BaseHTTPSHandler, self).__init__(**kwargs) self.session = FuturesSession(executor=VerboseThreadPoolExecutor( max_workers=2 # this is the default used by requests_futures )) self._add_auth_information() - @classmethod - def perform_test(cls, settings): - """ - Tests logging connectivity for the current logging settings. - @raises LoggingConnectivityException - """ - handler = cls.from_django_settings(settings) - handler.enabled_flag = True - handler.setFormatter(LogstashFormatter(settings_module=settings)) - logger = logging.getLogger(__file__) - fn, lno, func = logger.findCaller() - record = logger.makeRecord('awx', 10, fn, lno, - 'AWX Connection Test', tuple(), - None, func) - futures = handler.emit(record) - for future in futures: - try: - resp = future.result() - if not resp.ok: - raise LoggingConnectivityException( - ': '.join([str(resp.status_code), resp.reason or '']) - ) - except RequestException as e: - raise LoggingConnectivityException(str(e)) - def _get_post_kwargs(self, payload_input): if self.message_type == 'splunk': # Splunk needs data nested under key "event" @@ -265,6 +217,10 @@ def _encode_payload_for_socket(payload): class TCPHandler(BaseHandler): + def __init__(self, tcp_timeout=5, **kwargs): + self.tcp_timeout = tcp_timeout + super(TCPHandler, self).__init__(**kwargs) + def _send(self, payload): payload = _encode_payload_for_socket(payload) sok = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -273,39 +229,32 @@ class TCPHandler(BaseHandler): sok.setblocking(0) _, ready_to_send, _ = select.select([], [sok], [], float(self.tcp_timeout)) if len(ready_to_send) == 0: - logger.warning("Socket currently busy, failed to send message") - sok.close() - return - sok.send(payload) + ret = SocketResult(False, "Socket currently busy, failed to send message") + logger.warning(ret.reason) + else: + sok.send(payload) + ret = SocketResult(True) # success! except Exception as e: - logger.exception("Error sending message from %s: %s" % - (TCPHandler.__name__, e.message)) - sok.close() + ret = SocketResult(False, "Error sending message from %s: %s" % + (TCPHandler.__name__, + ' '.join(six.text_type(arg) for arg in e.args))) + logger.exception(ret.reason) + finally: + sok.close() + return ret class UDPHandler(BaseHandler): + message = "Cannot determine if UDP messages are received." + def __init__(self, **kwargs): super(UDPHandler, self).__init__(**kwargs) self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) def _send(self, payload): payload = _encode_payload_for_socket(payload) - return self.socket.sendto(payload, (self._get_host(hostname_only=True), self.port or 0)) - - @classmethod - def perform_test(cls, settings): - """ - Tests logging connectivity for the current logging settings. - """ - handler = cls.from_django_settings(settings) - handler.enabled_flag = True - handler.setFormatter(LogstashFormatter(settings_module=settings)) - logger = logging.getLogger(__file__) - fn, lno, func = logger.findCaller() - record = logger.makeRecord('awx', 10, fn, lno, - 'AWX Connection Test', tuple(), - None, func) - handler.emit(_encode_payload_for_socket(record)) + self.socket.sendto(payload, (self._get_host(hostname_only=True), self.port or 0)) + return SocketResult(True, reason=self.message) HANDLER_MAPPING = { @@ -315,6 +264,88 @@ HANDLER_MAPPING = { } +class AWXProxyHandler(logging.Handler): + ''' + Handler specific to the AWX external logging feature + + Will dynamically create a handler specific to the configured + protocol, and will create a new one automatically on setting change + + Managing parameters: + All parameters will get their value from settings as a default + if the parameter was either provided on init, or set manually, + this value will take precedence. + Parameters match same parameters in the actualized handler classes. + ''' + + def __init__(self, **kwargs): + # TODO: process 'level' kwarg + super(AWXProxyHandler, self).__init__(**kwargs) + self._handler = None + self._old_kwargs = {} + + def get_handler_class(self, protocol): + return HANDLER_MAPPING[protocol] + + def get_handler(self, custom_settings=None, force_create=False): + new_kwargs = {} + use_settings = custom_settings or settings + for field_name, setting_name in PARAM_NAMES.items(): + val = getattr(use_settings, setting_name, None) + if val is None: + continue + new_kwargs[field_name] = val + if new_kwargs == self._old_kwargs and self._handler and (not force_create): + # avoids re-creating session objects, and other such things + return self._handler + self._old_kwargs = new_kwargs.copy() + # TODO: remove any kwargs no applicable to that particular handler + protocol = new_kwargs.pop('protocol', None) + HandlerClass = self.get_handler_class(protocol) + # cleanup old handler and make new one + if self._handler: + self._handler.close() + logger.debug('Creating external log handler due to startup or settings change.') + self._handler = HandlerClass(**new_kwargs) + if self.formatter: + # self.format(record) is called inside of emit method + # so not safe to assume this can be handled within self + self._handler.setFormatter(self.formatter) + return self._handler + + def emit(self, record): + actual_handler = self.get_handler() + return actual_handler.emit(record) + + def perform_test(self, custom_settings): + """ + Tests logging connectivity for given settings module. + @raises LoggingConnectivityException + """ + handler = self.get_handler(custom_settings=custom_settings, force_create=True) + handler.setFormatter(LogstashFormatter()) + logger = logging.getLogger(__file__) + fn, lno, func = logger.findCaller() + record = logger.makeRecord('awx', 10, fn, lno, + 'AWX Connection Test', tuple(), + None, func) + futures = handler.emit(record) + for future in futures: + try: + resp = future.result() + if not resp.ok: + if isinstance(resp, SocketResult): + raise LoggingConnectivityException( + 'Socket error: {}'.format(resp.reason or '') + ) + else: + raise LoggingConnectivityException( + ': '.join([str(resp.status_code), resp.reason or '']) + ) + except RequestException as e: + raise LoggingConnectivityException(str(e)) + + ColorHandler = logging.StreamHandler if settings.COLOR_LOGS is True: @@ -340,41 +371,3 @@ if settings.COLOR_LOGS is True: except ImportError: # logutils is only used for colored logs in the dev environment pass - - -def _add_or_remove_logger(address, instance): - specific_logger = logging.getLogger(address) - for i, handler in enumerate(specific_logger.handlers): - if isinstance(handler, (HTTPSNullHandler, BaseHTTPSHandler)): - specific_logger.handlers[i] = instance or HTTPSNullHandler() - break - else: - if instance is not None: - specific_logger.handlers.append(instance) - - -def configure_external_logger(settings_module, is_startup=True): - is_enabled = settings_module.LOG_AGGREGATOR_ENABLED - if is_startup and (not is_enabled): - # Pass-through if external logging not being used - return - - instance = None - if is_enabled: - handler_class = HANDLER_MAPPING[settings_module.LOG_AGGREGATOR_PROTOCOL] - instance = handler_class.from_django_settings(settings_module) - - # Obtain the Formatter class from settings to maintain customizations - configurator = logging.config.DictConfigurator(settings_module.LOGGING) - formatter_config = settings_module.LOGGING['formatters']['json'].copy() - formatter_config['settings_module'] = settings_module - formatter = configurator.configure_custom(formatter_config) - - instance.setFormatter(formatter) - - awx_logger_instance = instance - if is_enabled and 'awx' not in settings_module.LOG_AGGREGATOR_LOGGERS: - awx_logger_instance = None - - _add_or_remove_logger('awx.analytics', instance) - _add_or_remove_logger('awx', awx_logger_instance) diff --git a/awx/main/utils/reload.py b/awx/main/utils/reload.py index 6da8b0b4f7..8da1fb0096 100644 --- a/awx/main/utils/reload.py +++ b/awx/main/utils/reload.py @@ -8,29 +8,9 @@ import logging # Django from django.conf import settings -# Celery -from celery import Celery - logger = logging.getLogger('awx.main.utils.reload') -def _uwsgi_fifo_command(uwsgi_command): - # http://uwsgi-docs.readthedocs.io/en/latest/MasterFIFO.html#available-commands - logger.warn('Initiating uWSGI chain reload of server') - TRIGGER_COMMAND = uwsgi_command - with open(settings.UWSGI_FIFO_LOCATION, 'w') as awxfifo: - awxfifo.write(TRIGGER_COMMAND) - - -def _reset_celery_thread_pool(): - # Do not use current_app because of this outstanding issue: - # https://github.com/celery/celery/issues/4410 - app = Celery('awx') - app.config_from_object('django.conf:settings') - app.control.broadcast('pool_restart', arguments={'reload': True}, - destination=['celery@{}'.format(settings.CLUSTER_HOST_ID)], reply=False) - - def _supervisor_service_command(service_internal_names, command, communicate=True): ''' Service internal name options: @@ -68,21 +48,6 @@ def _supervisor_service_command(service_internal_names, command, communicate=Tru logger.info('Submitted supervisorctl {} command, not waiting for result'.format(command)) -def restart_local_services(service_internal_names): - logger.warn('Restarting services {} on this node in response to user action'.format(service_internal_names)) - if 'uwsgi' in service_internal_names: - _uwsgi_fifo_command(uwsgi_command='c') - service_internal_names.remove('uwsgi') - restart_celery = False - if 'celery' in service_internal_names: - restart_celery = True - service_internal_names.remove('celery') - _supervisor_service_command(service_internal_names, command='restart') - if restart_celery: - # Celery restarted last because this probably includes current process - _reset_celery_thread_pool() - - def stop_local_services(service_internal_names, communicate=True): logger.warn('Stopping services {} on this node in response to user action'.format(service_internal_names)) _supervisor_service_command(service_internal_names, command='stop', communicate=communicate) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 20c8a82a1f..4a7fc96358 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -1001,6 +1001,9 @@ LOGGING = { 'require_debug_true_or_test': { '()': 'awx.main.utils.RequireDebugTrueOrTest', }, + 'external_log_enabled': { + '()': 'awx.main.utils.filters.ExternalLoggerEnabled' + }, }, 'formatters': { 'simple': { @@ -1034,11 +1037,10 @@ LOGGING = { 'class': 'logging.NullHandler', 'formatter': 'simple', }, - 'http_receiver': { - 'class': 'awx.main.utils.handlers.HTTPSNullHandler', - 'level': 'DEBUG', + 'external_logger': { + 'class': 'awx.main.utils.handlers.AWXProxyHandler', 'formatter': 'json', - 'host': '', + 'filters': ['external_log_enabled'], }, 'mail_admins': { 'level': 'ERROR', @@ -1131,7 +1133,7 @@ LOGGING = { 'handlers': ['console'], }, 'awx': { - 'handlers': ['console', 'file', 'tower_warnings'], + 'handlers': ['console', 'file', 'tower_warnings', 'external_logger'], 'level': 'DEBUG', }, 'awx.conf': { @@ -1179,7 +1181,7 @@ LOGGING = { 'propagate': False, }, 'awx.analytics': { - 'handlers': ['http_receiver'], + 'handlers': ['external_logger'], 'level': 'INFO', 'propagate': False }, diff --git a/awx/settings/development.py b/awx/settings/development.py index eec3689ee7..9f1d916bc0 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -149,8 +149,6 @@ SERVICE_NAME_DICT = { "uwsgi": "uwsgi", "daphne": "daphne", "nginx": "nginx"} -# Used for sending commands in automatic restart -UWSGI_FIFO_LOCATION = '/awxfifo' try: socket.gethostbyname('docker.for.mac.internal') diff --git a/awx/settings/production.py b/awx/settings/production.py index d1d5baa3ad..ff7d966138 100644 --- a/awx/settings/production.py +++ b/awx/settings/production.py @@ -68,8 +68,6 @@ SERVICE_NAME_DICT = { "channels": "awx-channels-worker", "uwsgi": "awx-uwsgi", "daphne": "awx-daphne"} -# Used for sending commands in automatic restart -UWSGI_FIFO_LOCATION = '/var/lib/awx/awxfifo' # Store a snapshot of default settings at this point before loading any # customizable config files.