Merge pull request #5239 from AlanCoding/migration_cache

Reduce API response times by 30% by using memcache migration flag

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

awx/main/apps.py

@@ -1,8 +1,17 @@
 from django.apps import AppConfig
+from django.db.models.signals import pre_migrate
 from django.utils.translation import ugettext_lazy as _
 
 
+def raise_migration_flag(**kwargs):
+    from awx.main.tasks import set_migration_flag
+    set_migration_flag.delay()
+
+
 class MainConfig(AppConfig):
 
     name = 'awx.main'
     verbose_name = _('Main')
+
+    def ready(self):
+        pre_migrate.connect(raise_migration_flag, sender=self)

awx/main/middleware.py

@@ -13,8 +13,7 @@ import urllib.parse
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.db.models.signals import post_save
-from django.db.migrations.executor import MigrationExecutor
-from django.db import IntegrityError, connection
+from django.db import IntegrityError
 from django.utils.functional import curry
 from django.shortcuts import get_object_or_404, redirect
 from django.apps import apps
@@ -24,6 +23,7 @@ from django.urls import reverse, resolve
 
 from awx.main.models import ActivityStream
 from awx.main.utils.named_url_graph import generate_graph, GraphNode
+from awx.main.utils.db import migration_in_progress_check_or_relase
 from awx.conf import fields, register
 
 
@@ -213,8 +213,7 @@ class URLModificationMiddleware(MiddlewareMixin):
 class MigrationRanCheckMiddleware(MiddlewareMixin):
 
     def process_request(self, request):
-        executor = MigrationExecutor(connection)
-        plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
-        if bool(plan) and \
-                getattr(resolve(request.path), 'url_name', '') != 'migrations_notran':
+        if migration_in_progress_check_or_relase():
+            if getattr(resolve(request.path), 'url_name', '') == 'migrations_notran':
+                return
             return redirect(reverse("ui:migrations_notran"))

awx/main/scheduler/tasks.py

@@ -5,11 +5,15 @@ import logging
 # AWX
 from awx.main.scheduler import TaskManager
 from awx.main.dispatch.publish import task
+from awx.main.utils.db import migration_in_progress_check_or_relase
 
 logger = logging.getLogger('awx.main.scheduler')
 
 
 @task()
 def run_task_manager():
+    if migration_in_progress_check_or_relase():
+        logger.debug("Not running task manager because migration is in progress.")
+        return
     logger.debug("Running Tower task manager.")
     TaskManager().schedule()

awx/main/tasks.py

@@ -263,6 +263,12 @@ def apply_cluster_membership_policies():
         logger.debug('Cluster policy computation finished in {} seconds'.format(time.time() - started_compute))
 
 
+@task(queue='tower_broadcast_all', exchange_type='fanout')
+def set_migration_flag():
+    logger.debug('Received migration-in-progress signal, will serve redirect.')
+    cache.set('migration_in_progress', True)
+
+
 @task(queue='tower_broadcast_all', exchange_type='fanout')
 def handle_setting_changes(setting_keys):
     orig_len = len(setting_keys)

awx/main/utils/db.py

@@ -1,8 +1,16 @@
 # Copyright (c) 2017 Ansible by Red Hat
 # All Rights Reserved.
 
+import logging
 from itertools import chain
 
+from django.core.cache import cache
+from django.db.migrations.executor import MigrationExecutor
+from django.db import connection
+
+
+logger = logging.getLogger('awx.main.utils.db')
+
 
 def get_all_field_names(model):
     # Implements compatibility with _meta.get_all_field_names
@@ -14,3 +22,21 @@ def get_all_field_names(model):
         # GenericForeignKey from the results.
         if not (field.many_to_one and field.related_model is None)
     )))
+
+
+def migration_in_progress_check_or_relase():
+    '''A memcache flag is raised (set to True) to inform cluster
+    that a migration is ongoing see main.apps.MainConfig.ready
+    if the flag is True then the flag is removed on this instance if
+        models-db consistency is observed
+    effective value of migration flag is returned
+    '''
+    migration_in_progress = cache.get('migration_in_progress', False)
+    if migration_in_progress:
+        executor = MigrationExecutor(connection)
+        plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
+        if not bool(plan):
+            logger.info('Detected that migration finished, migration flag taken down.')
+            cache.delete('migration_in_progress')
+            migration_in_progress = False
+    return migration_in_progress