Check connection for scan jobs before actually running the job, fail

early if the system tracking database isn't available

awx/fact/utils/connection.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2015 Ansible, Inc. (formerly AnsibleWorks, Inc.)
+# All Rights Reserved.
+
+from django.conf import settings
+from mongoengine import connect
+from mongoengine.connection import get_db, ConnectionError
+
+def test_mongo_connection():
+    # Connect to Mongo
+    try:
+        # Sanity check: If we have intentionally invalid settings, then we
+        # know we cannot connect.
+        if settings.MONGO_HOST == NotImplemented:
+            raise ConnectionError
+
+        # Attempt to connect to the MongoDB database.
+        connect(settings.MONGO_DB,
+                host=settings.MONGO_HOST,
+                port=int(settings.MONGO_PORT),
+                username=settings.MONGO_USERNAME,
+                password=settings.MONGO_PASSWORD,
+                tz_aware=settings.USE_TZ)
+        return True
+    except ConnectionError:
+        return False
+    

awx/main/management/commands/run_fact_cache_receiver.py

@@ -2,6 +2,7 @@
 # All Rights Reserved
 
 import logging
+import sys
 from datetime import datetime
 
 from django.core.management.base import NoArgsCommand
@@ -52,7 +53,7 @@ class FactCacheReceiver(object):
             logger.warn('Database inconsistent. Multiple FactHost "%s" exist. Try the query %s to find the records.' % (hostname, query))
             return
         except Exception, e:
-            logger.error("Exception communicating with Mongo: %s" % str(e))
+            logger.error("Exception communicating with Fact Cache Database: %s" % str(e))
             return
 
         (module, facts) = self.process_facts(facts_data)

awx/main/tasks.py

@@ -43,6 +43,7 @@ from awx.main.queue import FifoQueue
 from awx.main.utils import (get_ansible_version, decrypt_field, update_scm_url,
                             ignore_inventory_computed_fields, emit_websocket_notification,
                             check_proot_installed, build_proot_temp_dir, wrap_args_with_proot)
+from awx.fact.utils.connection import test_mongo_connection
 
 __all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate',
            'RunAdHocCommand', 'handle_work_error', 'update_inventory_computed_fields']
@@ -436,6 +437,11 @@ class BaseTask(Task):
         else:
             return 'failed', child.exitstatus
 
+    def pre_run_hook(self, instance, **kwargs):
+        '''
+        Hook for any steps to run before the job/task starts
+        '''
+
     def post_run_hook(self, instance, **kwargs):
         '''
         Hook for any steps to run after job/task is complete.
@@ -451,6 +457,7 @@ class BaseTask(Task):
         status, rc, tb = 'error', None, ''
         output_replacements = []
         try:
+            self.pre_run_hook(instance, **kwargs)
             if instance.cancel_flag:
                 instance = self.update_model(instance.pk, status='canceled')
             if instance.status != 'running':
@@ -766,6 +773,14 @@ class RunJob(BaseTask):
         '''
         return getattr(settings, 'AWX_PROOT_ENABLED', False)
 
+    def pre_run_hook(self, job, **kwargs):
+        print("In pre-run")
+        if job.job_type == PERM_INVENTORY_SCAN:
+            print("In scan")
+            if not test_mongo_connection():
+                print("Mongo isn't running")
+                raise RuntimeError("Fact Scan Database is offline")
+
     def post_run_hook(self, job, **kwargs):
         '''
         Hook for actions to run after job/task has completed.