Add automatic deprovisioning support, only enabled for openshift

* Implement a config watcher for service restarts
* If the configmap bind point changes then restart all services

awx/main/tasks.py

@@ -292,6 +292,10 @@ def cluster_node_heartbeat(self):
             other_inst.save(update_fields=['capacity'])
             logger.error("Host {} last checked in at {}, marked as lost.".format(
                 other_inst.hostname, other_inst.modified))
+            if settings.AWX_AUTO_DEPROVISION_INSTANCES:
+                deprovision_hostname = other_inst.hostname
+                other_inst.delete()
+                logger.info("Host {} Automatically Deprovisioned.".format(deprovision_hostname))
         except DatabaseError as e:
             if 'did not affect any rows' in str(e):
                 logger.debug('Another instance has marked {} as lost'.format(other_inst.hostname))

awx/settings/defaults.py

@@ -641,6 +641,9 @@ AWX_ANSIBLE_CALLBACK_PLUGINS = ""
 # Time at which an HA node is considered active
 AWX_ACTIVE_NODE_TIME = 7200
 
+# Automatically remove nodes that have missed their heartbeats after some time
+AWX_AUTO_DEPROVISION_INSTANCES = False
+
 # Enable Pendo on the UI, possible values are 'off', 'anonymous', and 'detailed'
 # Note: This setting may be overridden by database settings.
 PENDO_TRACKING_STATE = "off"