Respect settings to keep files and work units

Add new logic to cleanup orphaned work units
  from administrative tasks

Remove noisy log which is often irrelevant
  about running-cleanup-on-execution-nodes
  we already have other logs for this

awx/main/utils/receptor.py

@@ -3,6 +3,7 @@ import yaml
 import time
 
 from receptorctl.socket_interface import ReceptorControl
+
 from django.conf import settings
 
 from enum import Enum, unique
@@ -11,6 +12,8 @@ logger = logging.getLogger('awx.main.utils.receptor')
 
 __RECEPTOR_CONF = '/etc/receptor/receptor.conf'
 
+RECEPTOR_ACTIVE_STATES = ('Pending', 'Running')
+
 
 @unique
 class ReceptorConnectionType(Enum):
@@ -62,6 +65,32 @@ def get_conn_type(node_name, receptor_ctl):
             return ReceptorConnectionType(node.get('ConnType'))
 
 
+def administrative_workunit_reaper(work_list=None):
+    """
+    This releases completed work units that were spawned by actions inside of this module
+    specifically, this should catch any completed work unit left by
+     - worker_info
+     - worker_cleanup
+    These should ordinarily be released when the method finishes, but this is a
+    cleanup of last-resort, in case something went awry
+    """
+    receptor_ctl = get_receptor_ctl()
+    if work_list is None:
+        work_list = receptor_ctl.simple_command("work list")
+
+    for unit_id, work_data in work_list.items():
+        extra_data = work_data.get('ExtraData')
+        if (extra_data is None) or (extra_data.get('RemoteWorkType') != 'ansible-runner'):
+            continue  # if this is not ansible-runner work, we do not want to touch it
+        params = extra_data.get('RemoteParams', {}).get('params')
+        if not (params == '--worker-info' or params.startswith('cleanup')):
+            continue  # if this is not a cleanup or health check, we do not want to touch it
+        if work_data.get('StateName') in RECEPTOR_ACTIVE_STATES:
+            continue  # do not want to touch active work units
+        logger.info(f'Reaping orphaned work unit {unit_id} with params {params}')
+        receptor_ctl.simple_command(f"work release {unit_id}")
+
+
 class RemoteJobError(RuntimeError):
     pass
 
@@ -95,7 +124,7 @@ def run_until_complete(node, timing_data=None, **kwargs):
         while run_timing < 20.0:
             status = receptor_ctl.simple_command(f'work status {unit_id}')
             state_name = status.get('StateName')
-            if state_name not in ('Pending', 'Running'):
+            if state_name not in RECEPTOR_ACTIVE_STATES:
                 break
             run_timing = time.time() - run_start
             time.sleep(0.5)
@@ -110,9 +139,10 @@ def run_until_complete(node, timing_data=None, **kwargs):
 
     finally:
 
-        res = receptor_ctl.simple_command(f"work release {unit_id}")
-        if res != {'released': unit_id}:
-            logger.warn(f'Could not confirm release of receptor work unit id {unit_id} from {node}, data: {res}')
+        if settings.RECEPTOR_RELEASE_WORK:
+            res = receptor_ctl.simple_command(f"work release {unit_id}")
+            if res != {'released': unit_id}:
+                logger.warn(f'Could not confirm release of receptor work unit id {unit_id} from {node}, data: {res}')
 
         receptor_ctl.close()