AAP-89010: Guard heartbeat against missing receptor config (#16611)

The EE sidecar container creates /etc/receptor/receptor.conf on pod
startup. If the task container's dispatcherd reaches the heartbeat
before the config exists, the instance rejoins the cluster at full
capacity with no receptor — causing fatal job failures with no
recovery path.

Three layers of defense:

1. dispatcherd management command: poll for the receptor config file
   before entering the main loop (10s timeout, exits with error to
   let k8s restart the container).

2. Heartbeat: if get_receptor_ctl() raises FileNotFoundError (config
   or socket missing), mark the instance offline so no jobs are
   dispatched to it. Previously this was silently ignored and the
   instance kept full capacity.

3. should_update_config(): catch FileNotFoundError and return True so
   write_receptor_config() can create the file on first boot instead
   of crashing.

Prior art: #12698 (closed without merge) took a broader approach with
custom exception types. See PR comment for overlap analysis.

Closes: AAP-89010

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

awx/main/management/commands/dispatcherd.py

@@ -6,6 +6,7 @@ import json
 import logging
 import logging.config
 import os
+import time
 
 from django.conf import settings
 from django.core.cache import cache as django_cache
@@ -15,6 +16,7 @@ from django.db import connection
 from dispatcherd.config import setup as dispatcher_setup
 
 from awx.main.dispatch.config import get_dispatcherd_config
+from awx.main.tasks.receptor import receptor_config_exists
 
 logger = logging.getLogger('awx.main.dispatch')
 
@@ -52,6 +54,14 @@ class Command(BaseCommand):
     def handle(self, *arg, **options):
         ensure_no_dispatcherd_env_config()
 
+        for attempt in range(5):
+            if receptor_config_exists():
+                break
+            logger.info("Waiting for receptor config... (attempt %d/5)", attempt + 1)
+            time.sleep(2)
+        else:
+            raise CommandError("Receptor config not found after 10s")
+
         self.configure_dispatcher_logging()
         config = get_dispatcherd_config(for_service=True)
         config_hash = _hash_config(config)

awx/main/tasks/receptor.py

@@ -117,6 +117,10 @@ class FuzzyError:
         return UnknownError(msg, node=node, state_name=state_name)
 
 
+def receptor_config_exists():
+    return os.path.exists(__RECEPTOR_CONF)
+
+
 def read_receptor_config():
     # for K8S deployments, getting a lock is necessary as another process
     # may be re-writing the config at this time
@@ -801,7 +805,11 @@ def should_update_config(new_config):
     tcp-peers in the config
     '''
 
-    current_config = read_receptor_config()  # this gets receptor conf lock
+    try:
+        current_config = read_receptor_config()  # this gets receptor conf lock
+    except FileNotFoundError:
+        logger.warning("Receptor config file not found, config needs to be written.")
+        return True
     for config_entry in current_config:
         if config_entry not in new_config:
             logger.warning(f"{config_entry} should not be in receptor config. Updating.")

awx/main/tasks/system.py

@@ -72,7 +72,13 @@ from awx.main.models import (
 from awx.main.models.credential import CredentialType
 from awx.main.tasks.helpers import is_run_threshold_reached
 from awx.main.tasks.host_indirect import save_indirect_host_entries
-from awx.main.tasks.receptor import administrative_workunit_reaper, get_receptor_ctl, worker_cleanup, worker_info, write_receptor_config
+from awx.main.tasks.receptor import (
+    administrative_workunit_reaper,
+    get_receptor_ctl,
+    worker_cleanup,
+    worker_info,
+    write_receptor_config,
+)
 from awx.main.utils.common import ignore_inventory_computed_fields, ignore_inventory_group_removal
 from awx.main.utils.migration import is_database_synchronized
 from awx.main.utils.reload import stop_local_services
@@ -582,16 +588,11 @@ def inspect_established_receptor_connections(mesh_status):
     InstanceLink.objects.bulk_update(update_links, ['link_state'])
 
 
-def inspect_execution_and_hop_nodes(instance_list):
+def inspect_execution_and_hop_nodes(instance_list, receptor_ctl):
     with advisory_lock('inspect_execution_and_hop_nodes_lock', wait=False):
         node_lookup = {inst.hostname: inst for inst in instance_list}
         try:
-            ctl = get_receptor_ctl()
-        except FileNotFoundError:
-            logger.error('Receptor daemon not running, skipping execution node check')
-            return
-        try:
-            mesh_status = ctl.simple_command('status')
+            mesh_status = receptor_ctl.simple_command('status')
         except ValueError as exc:
             logger.error(f'Error running receptorctl status command, error: {str(exc)}')
             return
@@ -727,7 +728,16 @@ def _heartbeat_instance_management():
             this_inst = inst
             break
 
-    inspect_execution_and_hop_nodes(instance_list)
+    try:
+        ctl = get_receptor_ctl()
+    except FileNotFoundError:
+        logger.error('Receptor not available, marking instance offline.')
+        if this_inst:
+            this_inst.local_health_check()
+            this_inst.mark_offline(errors='Receptor not available')
+        return None, None, None
+
+    inspect_execution_and_hop_nodes(instance_list, ctl)
 
     for inst in list(instance_list):
         if inst == this_inst:

awx/main/tests/unit/tasks/test_system.py

@@ -1,7 +1,7 @@
 import pytest
 from unittest.mock import MagicMock, patch
-from awx.main.tasks.system import update_inventory_computed_fields
-from awx.main.models import Inventory
+from awx.main.tasks.system import _heartbeat_instance_management, update_inventory_computed_fields
+from awx.main.models import Instance, Inventory
 from django.db import DatabaseError
 
 
@@ -62,3 +62,19 @@ def test_update_inventory_computed_fields_database_error_nosqlstate(mock_logger,
             mock_filter.assert_called_once_with(id=1)
             mock_update_computed_fields.assert_called_once()
             mock_inventory.update_computed_fields.assert_called_once()
+
+
+@patch('awx.main.tasks.system.get_receptor_ctl', side_effect=FileNotFoundError)
+@patch('awx.main.tasks.system.Instance.objects.filter')
+def test_heartbeat_marks_offline_when_receptor_unavailable(mock_filter, mock_get_ctl):
+    this_inst = MagicMock(spec=Instance)
+    this_inst.hostname = 'test-host'
+    mock_filter.return_value = [this_inst]
+
+    with patch('awx.main.tasks.system.settings') as mock_settings:
+        mock_settings.CLUSTER_HOST_ID = 'test-host'
+        result = _heartbeat_instance_management()
+
+    assert result == (None, None, None)
+    this_inst.local_health_check.assert_called_once()
+    this_inst.mark_offline.assert_called_once_with(errors='Receptor not available')

awx/main/tests/unit/utils/test_receptor.py

@@ -1,4 +1,6 @@
-from awx.main.tasks.receptor import _convert_args_to_cli
+from unittest.mock import patch
+
+from awx.main.tasks.receptor import _convert_args_to_cli, should_update_config
 
 
 def test_file_cleanup_scenario():
@@ -20,3 +22,8 @@ def test_image_cleanup_scenario():
         ' '.join(args)
         == 'cleanup --remove-images "quay.invalid/foo/bar:latest" "quay.invalid/foo/bar:devel" --image-prune --process-isolation-executable=podman'
     )
+
+
+def test_should_update_config_missing_file():
+    with patch('awx.main.tasks.receptor.read_receptor_config', side_effect=FileNotFoundError):
+        assert should_update_config([{'node': {}}]) is True