From 891e06d6ef614f7de0ce2ebab678f5331056e74b Mon Sep 17 00:00:00 2001 From: Chris Church Date: Tue, 23 Feb 2016 16:23:19 -0500 Subject: [PATCH 1/3] Add config for using Redis as cache, add memoize function to store results in cache. --- awx/api/license.py | 2 ++ awx/main/utils.py | 28 +++++++++++++++++++++++++--- awx/settings/defaults.py | 8 ++++++++ awx/settings/postprocess.py | 3 +++ docs/licenses/django-redis-cache.txt | 24 ++++++++++++++++++++++++ requirements/requirements.txt | 1 + 6 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 docs/licenses/django-redis-cache.txt diff --git a/awx/api/license.py b/awx/api/license.py index 55706364f8..1b225e3a1c 100644 --- a/awx/api/license.py +++ b/awx/api/license.py @@ -4,6 +4,7 @@ from rest_framework.exceptions import APIException from awx.main.task_engine import TaskSerializer +from awx.main.utils import memoize class LicenseForbids(APIException): @@ -11,6 +12,7 @@ class LicenseForbids(APIException): default_detail = 'Your Tower license does not allow that.' +@memoize() def get_license(show_key=False, bypass_database=False): """Return a dictionary representing the license currently in place on this Tower instance. diff --git a/awx/main/utils.py b/awx/main/utils.py index f1a71fbcfc..63235ffca3 100644 --- a/awx/main/utils.py +++ b/awx/main/utils.py @@ -16,9 +16,13 @@ import threading import contextlib import tempfile +# Decorator +from decorator import decorator + # Django REST Framework from rest_framework.exceptions import ParseError, PermissionDenied from django.utils.encoding import smart_str +from django.utils.text import slugify from django.core.urlresolvers import reverse from django.apps import apps @@ -27,7 +31,7 @@ from Crypto.Cipher import AES logger = logging.getLogger('awx.main.utils') -__all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore', +__all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore', 'memoize', 'get_ansible_version', 'get_ssh_version', 'get_awx_version', 'update_scm_url', 'get_type_for_model', 'get_model_for_type', 'to_python_boolean', 'ignore_inventory_computed_fields', 'ignore_inventory_group_removal', @@ -93,6 +97,23 @@ class RequireDebugTrueOrTest(logging.Filter): return settings.DEBUG or 'test' in sys.argv +def memoize(ttl=60): + ''' + Decorator to wrap a function and cache its result. + ''' + from django.core.cache import cache + + def _memoizer(f, *args, **kwargs): + key = slugify('%s %r %r' % (f.__name__, args, kwargs)) + value = cache.get(key) + if value is None: + value = f(*args, **kwargs) + cache.set(key, value, ttl) + return value + return decorator(_memoizer) + + +@memoize() def get_ansible_version(): ''' Return Ansible version installed. @@ -101,11 +122,11 @@ def get_ansible_version(): proc = subprocess.Popen(['ansible', '--version'], stdout=subprocess.PIPE) result = proc.communicate()[0] - stripped_result = result.split('\n')[0].replace('ansible', '').strip() - return stripped_result + return result.split('\n')[0].replace('ansible', '').strip() except: return 'unknown' +@memoize() def get_ssh_version(): ''' Return SSH version installed. @@ -444,6 +465,7 @@ def ignore_inventory_group_removal(): finally: _inventory_updates.is_removing = previous_value +@memoize() def check_proot_installed(): ''' Check that proot is installed. diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index aa8f69866b..19eed98c31 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -351,6 +351,14 @@ CELERYBEAT_SCHEDULE = { }, } +# Use Redis as cache backend. +CACHES = { + 'default': { + 'BACKEND': 'redis_cache.RedisCache', + 'LOCATION': BROKER_URL, + }, +} + # Social Auth configuration. SOCIAL_AUTH_STRATEGY = 'social.strategies.django_strategy.DjangoStrategy' SOCIAL_AUTH_STORAGE = 'social.apps.django_app.default.models.DjangoStorage' diff --git a/awx/settings/postprocess.py b/awx/settings/postprocess.py index 544758e04f..6cbeec051a 100644 --- a/awx/settings/postprocess.py +++ b/awx/settings/postprocess.py @@ -32,3 +32,6 @@ if not all([SOCIAL_AUTH_SAML_SP_ENTITY_ID, SOCIAL_AUTH_SAML_SP_PUBLIC_CERT, if not AUTH_BASIC_ENABLED: REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] = [x for x in REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] if x != 'rest_framework.authentication.BasicAuthentication'] + +# Update cache to use celery broker URL defined in configuration files. +CACHES['default']['LOCATION'] = BROKER_URL diff --git a/docs/licenses/django-redis-cache.txt b/docs/licenses/django-redis-cache.txt new file mode 100644 index 0000000000..fc023da976 --- /dev/null +++ b/docs/licenses/django-redis-cache.txt @@ -0,0 +1,24 @@ +Copyright (c) 2015 Sean Bleier +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/requirements/requirements.txt b/requirements/requirements.txt index bdc9c6945d..fcc2c66acc 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -21,6 +21,7 @@ django-extensions==1.5.9 git+https://github.com/chrismeyersfsu/django-jsonbfield@fix-sqlite_serialization#egg=jsonbfield django-polymorphic==0.7.2 django-radius==1.0.0 +django-redis-cache==1.6.5 djangorestframework==3.3.2 djangorestframework-yaml==1.0.2 django-split-settings==0.1.1 From 0687168e5ef6e46d2f58c2474fed33dbd27f9285 Mon Sep 17 00:00:00 2001 From: Chris Church Date: Thu, 25 Feb 2016 15:31:53 -0500 Subject: [PATCH 2/3] Don't use Redis as cache when running tests. --- awx/settings/defaults.py | 21 ++++++++++++++------- awx/settings/postprocess.py | 3 ++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 19eed98c31..1098d709db 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -351,13 +351,20 @@ CELERYBEAT_SCHEDULE = { }, } -# Use Redis as cache backend. -CACHES = { - 'default': { - 'BACKEND': 'redis_cache.RedisCache', - 'LOCATION': BROKER_URL, - }, -} +# Use Redis as cache backend (except when testing). +if is_testing(): + CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, + } +else: + CACHES = { + 'default': { + 'BACKEND': 'redis_cache.RedisCache', + 'LOCATION': BROKER_URL, + }, + } # Social Auth configuration. SOCIAL_AUTH_STRATEGY = 'social.strategies.django_strategy.DjangoStrategy' diff --git a/awx/settings/postprocess.py b/awx/settings/postprocess.py index 6cbeec051a..d63833aac8 100644 --- a/awx/settings/postprocess.py +++ b/awx/settings/postprocess.py @@ -34,4 +34,5 @@ if not AUTH_BASIC_ENABLED: REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] = [x for x in REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] if x != 'rest_framework.authentication.BasicAuthentication'] # Update cache to use celery broker URL defined in configuration files. -CACHES['default']['LOCATION'] = BROKER_URL +if CACHES['default']['BACKEND'] == 'redis_cache.RedisCache': + CACHES['default']['LOCATION'] = BROKER_URL From 3bb14e2f72d520d711191df01bafd3424ecbb43c Mon Sep 17 00:00:00 2001 From: Chris Church Date: Sun, 6 Mar 2016 18:00:49 -0500 Subject: [PATCH 3/3] Clear cache between tests or when license is updated. --- awx/api/views.py | 5 +++++ awx/main/tests/base.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/awx/api/views.py b/awx/api/views.py index e75a5288ed..a2163da8c8 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -18,6 +18,7 @@ from collections import OrderedDict # Django from django.conf import settings from django.contrib.auth.models import User +from django.core.cache import cache from django.core.urlresolvers import reverse from django.core.exceptions import FieldError from django.db.models import Q, Count @@ -263,6 +264,8 @@ class ApiV1ConfigView(APIView): if license_data['valid_key']: tower_settings.LICENSE = data_actual tower_settings.TOWER_URL_BASE = "{}://{}".format(request.scheme, request.get_host()) + # Clear cache when license is updated. + cache.clear() return Response(license_data) return Response({"error": "Invalid license"}, status=status.HTTP_400_BAD_REQUEST) @@ -282,6 +285,8 @@ class ApiV1ConfigView(APIView): break TowerSettings.objects.filter(key="LICENSE").delete() + # Clear cache when license is updated. + cache.clear() # Only stop mongod if license removal succeeded if has_error is None: diff --git a/awx/main/tests/base.py b/awx/main/tests/base.py index cd3754b23f..63fa0ed571 100644 --- a/awx/main/tests/base.py +++ b/awx/main/tests/base.py @@ -24,6 +24,7 @@ import yaml import django.test from django.conf import settings, UserSettingsHolder from django.contrib.auth.models import User +from django.core.cache import cache from django.test.client import Client from django.test.utils import override_settings from django.utils.encoding import force_text @@ -152,6 +153,7 @@ class BaseTestMixin(QueueTestMixin, MockCommonlySlowTestMixin): 'LOCATION': 'unittests' } } + cache.clear() self._start_time = time.time() def tearDown(self): @@ -195,6 +197,7 @@ class BaseTestMixin(QueueTestMixin, MockCommonlySlowTestMixin): writer.write_file(license_path) self._temp_paths.append(license_path) os.environ['AWX_LICENSE_FILE'] = license_path + cache.clear() def create_basic_license_file(self, instance_count=100, license_date=int(time.time() + 3600)): writer = LicenseWriter( @@ -209,6 +212,7 @@ class BaseTestMixin(QueueTestMixin, MockCommonlySlowTestMixin): writer.write_file(license_path) self._temp_paths.append(license_path) os.environ['AWX_LICENSE_FILE'] = license_path + cache.clear() def create_expired_license_file(self, instance_count=1000, grace_period=False): license_date = time.time() - 1