Use the ansible-runner worker --worker-info to perform execution node capacity checks (#10825)

* Introduce utilities for --worker-info health check integration

* Handle case where ansible-runner is not installed

* Add ttl parameter for health check

* Reformulate return data structure and add lots of error cases

* Move up the cleanup tasks, close sockets

* Integrate new --worker-info into the execution node capacity check

* Undo the raw value override from the PoC

* Additional refinement to execution node check frequency

* Put in more complete network diagram

* Followup on comment to remove modified from from health check responsibilities

awx/main/utils/common.py

@@ -699,12 +699,25 @@ def parse_yaml_or_json(vars_str, silent_failure=True):
     return vars_dict
 
 
-def get_cpu_capacity(raw=None):
+def get_cpu_effective_capacity(cpu_count):
     from django.conf import settings
 
     settings_forkcpu = getattr(settings, 'SYSTEM_TASK_FORKS_CPU', None)
     env_forkcpu = os.getenv('SYSTEM_TASK_FORKS_CPU', None)
 
+    if env_forkcpu:
+        forkcpu = int(env_forkcpu)
+    elif settings_forkcpu:
+        forkcpu = int(settings_forkcpu)
+    else:
+        forkcpu = 4
+
+    return cpu_count * forkcpu
+
+
+def get_cpu_capacity():
+    from django.conf import settings
+
     settings_abscpu = getattr(settings, 'SYSTEM_TASK_ABS_CPU', None)
     env_abscpu = os.getenv('SYSTEM_TASK_ABS_CPU', None)
 
@@ -713,24 +726,30 @@ def get_cpu_capacity(raw=None):
     elif settings_abscpu is not None:
         return 0, int(settings_abscpu)
 
-    if raw is None:
-        raw = psutil.cpu_count()
+    cpu = psutil.cpu_count()
 
-    if env_forkcpu:
-        forkcpu = int(env_forkcpu)
-    elif settings_forkcpu:
-        forkcpu = int(settings_forkcpu)
-    else:
-        forkcpu = 4
-    return (raw, raw * forkcpu)
+    return (cpu, get_cpu_effective_capacity(cpu))
 
 
-def get_mem_capacity(raw_mb=None):
+def get_mem_effective_capacity(mem_mb):
     from django.conf import settings
 
     settings_forkmem = getattr(settings, 'SYSTEM_TASK_FORKS_MEM', None)
     env_forkmem = os.getenv('SYSTEM_TASK_FORKS_MEM', None)
 
+    if env_forkmem:
+        forkmem = int(env_forkmem)
+    elif settings_forkmem:
+        forkmem = int(settings_forkmem)
+    else:
+        forkmem = 100
+
+    return max(1, ((mem_mb // 1024 // 1024) - 2048) // forkmem)
+
+
+def get_mem_capacity():
+    from django.conf import settings
+
     settings_absmem = getattr(settings, 'SYSTEM_TASK_ABS_MEM', None)
     env_absmem = os.getenv('SYSTEM_TASK_ABS_MEM', None)
 
@@ -739,16 +758,8 @@ def get_mem_capacity(raw_mb=None):
     elif settings_absmem is not None:
         return 0, int(settings_absmem)
 
-    if env_forkmem:
-        forkmem = int(env_forkmem)
-    elif settings_forkmem:
-        forkmem = int(settings_forkmem)
-    else:
-        forkmem = 100
-
-    if raw_mb is None:
-        raw_mb = psutil.virtual_memory().total
-    return (raw_mb, max(1, ((raw_mb // 1024 // 1024) - 2048) // forkmem))
+    mem = psutil.virtual_memory().total
+    return (mem, get_mem_effective_capacity(mem))
 
 
 def get_system_task_capacity(scale=Decimal(1.0), cpu_capacity=None, mem_capacity=None):

awx/main/utils/receptor.py

@@ -0,0 +1,77 @@
+import logging
+import yaml
+import time
+
+from receptorctl.socket_interface import ReceptorControl
+
+
+logger = logging.getLogger('awx.main.utils.receptor')
+
+
+def get_receptor_ctl():
+    return ReceptorControl('/var/run/receptor/receptor.sock')
+
+
+def worker_info(node_name):
+    receptor_ctl = get_receptor_ctl()
+    transmit_start = time.time()
+    error_list = []
+    data = {'Errors': error_list, 'transmit_timing': 0.0}
+
+    result = receptor_ctl.submit_work(worktype='ansible-runner', payload='', params={"params": f"--worker-info"}, ttl='20s', node=node_name)
+
+    unit_id = result['unitid']
+    run_start = time.time()
+    data['transmit_timing'] = run_start - transmit_start
+    data['run_timing'] = 0.0
+
+    try:
+
+        resultfile = receptor_ctl.get_work_results(unit_id)
+
+        stdout = ''
+
+        while data['run_timing'] < 20.0:
+            status = receptor_ctl.simple_command(f'work status {unit_id}')
+            state_name = status.get('StateName')
+            if state_name not in ('Pending', 'Running'):
+                break
+            data['run_timing'] = time.time() - run_start
+            time.sleep(0.5)
+        else:
+            error_list.append(f'Timeout getting worker info on {node_name}, state remains in {state_name}')
+
+        stdout = resultfile.read()
+        stdout = str(stdout, encoding='utf-8')
+
+    finally:
+
+        res = receptor_ctl.simple_command(f"work release {unit_id}")
+        if res != {'released': unit_id}:
+            logger.warn(f'Could not confirm release of receptor work unit id {unit_id} from {node_name}, data: {res}')
+
+        receptor_ctl.close()
+
+    if state_name.lower() == 'failed':
+        work_detail = status.get('Detail', '')
+        if not work_detail.startswith('exit status'):
+            error_list.append(f'Receptor error getting worker info from {node_name}, detail:\n{work_detail}')
+        elif 'unrecognized arguments: --worker-info' in stdout:
+            error_list.append(f'Old version (2.0.1 or earlier) of ansible-runner on node {node_name} without --worker-info')
+        else:
+            error_list.append(f'Unknown ansible-runner error on node {node_name}, stdout:\n{stdout}')
+    else:
+        yaml_stdout = stdout.strip()
+        remote_data = {}
+        try:
+            remote_data = yaml.safe_load(yaml_stdout)
+        except Exception as json_e:
+            error_list.append(f'Failed to parse node {node_name} --worker-info output as YAML, error: {json_e}, data:\n{yaml_stdout}')
+
+        if not isinstance(remote_data, dict):
+            error_list.append(f'Remote node {node_name} --worker-info output is not a YAML dict, output:{stdout}')
+        else:
+            error_list.extend(remote_data.pop('Errors'))  # merge both error lists
+            data.update(remote_data)
+
+    return data