diff --git a/Makefile b/Makefile index 669b5eeeca..63b5ac259e 100644 --- a/Makefile +++ b/Makefile @@ -433,7 +433,7 @@ celeryd: @if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/tower/bin/activate; \ fi; \ - $(PYTHON) manage.py celeryd -l DEBUG -B --autoreload --autoscale=20,3 --schedule=$(CELERY_SCHEDULE_FILE) -Q projects,jobs,default,scheduler,$(COMPOSE_HOST) + $(PYTHON) manage.py celeryd -l DEBUG -B --autoreload --autoscale=20,3 --schedule=$(CELERY_SCHEDULE_FILE) -Q projects,jobs,default,scheduler,broadcast_all,$(COMPOSE_HOST) #$(PYTHON) manage.py celery multi show projects jobs default -l DEBUG -Q:projects projects -Q:jobs jobs -Q:default default -c:projects 1 -c:jobs 3 -c:default 3 -Ofair -B --schedule=$(CELERY_SCHEDULE_FILE) # Run to start the zeromq callback receiver diff --git a/awx/conf/signals.py b/awx/conf/signals.py index f2ae1baf16..a2a30b09c1 100644 --- a/awx/conf/signals.py +++ b/awx/conf/signals.py @@ -3,17 +3,16 @@ import logging # Django from django.conf import settings -from django.core.cache import cache from django.core.signals import setting_changed from django.db.models.signals import post_save, pre_delete, post_delete from django.dispatch import receiver -from django.utils.log import configure_logging # Tower import awx.main.signals from awx.conf import settings_registry from awx.conf.models import Setting from awx.conf.serializers import SettingSerializer +from awx.main.tasks import clear_cache_keys logger = logging.getLogger('awx.conf.signals') @@ -26,12 +25,16 @@ def handle_setting_change(key, for_delete=False): # When a setting changes or is deleted, remove its value from cache along # with any other settings that depend on it. setting_keys = [key] + setting_key_dict = {} + setting_key_dict[key] = key for dependent_key in settings_registry.get_dependent_settings(key): # Note: Doesn't handle multiple levels of dependencies! setting_keys.append(dependent_key) + setting_key_dict[dependent_key] = dependent_key cache_keys = set([Setting.get_cache_key(k) for k in setting_keys]) - logger.debug('cache delete_many(%r)', cache_keys) - cache.delete_many(cache_keys) + logger.debug('sending signals to delete cache keys(%r)', cache_keys) + # cache.delete_many(cache_keys) + clear_cache_keys.delay(setting_key_dict) # Send setting_changed signal with new value for each setting. for setting_key in setting_keys: @@ -41,11 +44,6 @@ def handle_setting_change(key, for_delete=False): value=getattr(settings, setting_key, None), enter=not bool(for_delete), ) - # TODO: Move logic to task to run on all cluster nodes - if setting_key.startswith('LOG_AGGREGATOR_'): - configure_logging(settings.LOGGING_CONFIG, settings.LOGGING) - # settings.LOGGING_CONFIG = None - # logging.config.dictConfig(settings.LOGGING) @receiver(post_save, sender=Setting) diff --git a/awx/main/log_utils/__init__.py b/awx/main/log_utils/__init__.py deleted file mode 100644 index 6c3f484790..0000000000 --- a/awx/main/log_utils/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) 2017 Ansible by Red Hat -# All Rights Reserved. diff --git a/awx/main/log_utils/utils.py b/awx/main/log_utils/utils.py deleted file mode 100644 index a57515888f..0000000000 --- a/awx/main/log_utils/utils.py +++ /dev/null @@ -1,57 +0,0 @@ -import os -import yaml - - -def parse_config_file(): - """ - Find the .splunk_logger config file in the current directory, or in the - user's home and parse it. The one in the current directory has precedence. - - :return: A tuple with: - - project_id - - access_token - """ - for filename in ('.splunk_logger', os.path.expanduser('~/.splunk_logger')): - - project_id, access_token, api_domain = _parse_config_file_impl(filename) - - if project_id is not None \ - and access_token is not None \ - and api_domain is not None: - return project_id, access_token, api_domain - - else: - return None, None, None - - -def _parse_config_file_impl(filename): - """ - Format for the file is: - - credentials: - project_id: ... - access_token: ... - api_domain: ... - - :param filename: The filename to parse - :return: A tuple with: - - project_id - - access_token - - api_domain - """ - try: - doc = yaml.load(file(filename).read()) - - project_id = doc["credentials"]["project_id"] - access_token = doc["credentials"]["access_token"] - api_domain = doc["credentials"]["api_domain"] - - return project_id, access_token, api_domain - except: - return None, None, None - - -def get_config_from_env(): - return (os.environ.get('SPLUNK_PROJECT_ID', None), - os.environ.get('SPLUNK_ACCESS_TOKEN', None), - os.environ.get('SPLUNK_API_DOMAIN', None)) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 0c8aa8acce..4f6e647b0c 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -42,6 +42,8 @@ from django.utils.encoding import smart_str from django.core.mail import send_mail from django.contrib.auth.models import User from django.utils.translation import ugettext_lazy as _ +from django.core.cache import cache +from django.utils.log import configure_logging # AWX from awx.main.constants import CLOUD_PROVIDERS @@ -83,6 +85,17 @@ def celery_startup(conf=None, **kwargs): logger.error("Failed to rebuild schedule {}: {}".format(sch, e)) +@task(queue='broadcast_all') +def clear_cache_keys(cache_keys): + set_of_keys = set([key for key in cache_keys]) + logger.debug('cache delete_many(%r)', set_of_keys) + cache.delete_many(set_of_keys) + for setting_key in set_of_keys: + if setting_key.startswith('LOG_AGGREGATOR_'): + configure_logging(settings.LOGGING_CONFIG, settings.LOGGING) + break + + @task(queue='default') def send_notifications(notification_list, job_id=None): if not isinstance(notification_list, list): diff --git a/awx/main/utils/__init__.py b/awx/main/utils/__init__.py new file mode 100644 index 0000000000..45b0a0131a --- /dev/null +++ b/awx/main/utils/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2017 Ansible Tower by Red Hat +# All Rights Reserved. + +# AWX +from awx.main.utils.common import * # noqa + +# Fields that didn't get included in __all__ +# TODO: after initial commit of file move to devel, these can be added +# to common.py __all__ and removed here +from awx.main.utils.common import ( # noqa + RequireDebugTrueOrTest, + encrypt_field, + parse_yaml_or_json, + decrypt_field, + build_url, + timestamp_apiformat, + model_instance_diff, + model_to_dict, + check_proot_installed, + build_proot_temp_dir, + wrap_args_with_proot, + get_system_task_capacity, + decrypt_field_value +) + diff --git a/awx/main/utils.py b/awx/main/utils/common.py similarity index 100% rename from awx/main/utils.py rename to awx/main/utils/common.py diff --git a/awx/main/log_utils/formatters.py b/awx/main/utils/formatters.py similarity index 100% rename from awx/main/log_utils/formatters.py rename to awx/main/utils/formatters.py diff --git a/awx/main/log_utils/handlers.py b/awx/main/utils/handlers.py similarity index 100% rename from awx/main/log_utils/handlers.py rename to awx/main/utils/handlers.py diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 1be3f2ca89..1174de726e 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -9,6 +9,7 @@ import djcelery from datetime import timedelta from kombu import Queue, Exchange +from kombu.common import Broadcast # global settings from django.conf import global_settings @@ -373,6 +374,7 @@ CELERY_QUEUES = ( Queue('default', Exchange('default'), routing_key='default'), Queue('jobs', Exchange('jobs'), routing_key='jobs'), Queue('scheduler', Exchange('scheduler', type='topic'), routing_key='scheduler.job.#', durable=False), + Broadcast('broadcast_all') # Projects use a fanout queue, this isn't super well supported ) CELERY_ROUTES = {'awx.main.tasks.run_job': {'queue': 'jobs', @@ -843,7 +845,7 @@ LOGGING = { 'format': '%(asctime)s %(levelname)-8s %(name)s %(message)s', }, 'json': { - '()': 'awx.main.log_utils.formatters.LogstashFormatter' + '()': 'awx.main.utils.formatters.LogstashFormatter' } }, 'handlers': { @@ -867,7 +869,7 @@ LOGGING = { 'formatter': 'simple', }, 'http_receiver': { - 'class': 'awx.main.log_utils.handlers.HTTPSHandler', + 'class': 'awx.main.utils.handlers.HTTPSHandler', 'level': 'INFO', 'formatter': 'json', 'host': '',