From 3ab18fd58f7fb8204f0314e343453781c1c2347c Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 21 Aug 2026 12:39:42 -0400 Subject: [PATCH] AAP-89010: Guard heartbeat against missing receptor config (#16611) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EE sidecar container creates /etc/receptor/receptor.conf on pod startup. If the task container's dispatcherd reaches the heartbeat before the config exists, the instance rejoins the cluster at full capacity with no receptor — causing fatal job failures with no recovery path. Three layers of defense: 1. dispatcherd management command: poll for the receptor config file before entering the main loop (10s timeout, exits with error to let k8s restart the container). 2. Heartbeat: if get_receptor_ctl() raises FileNotFoundError (config or socket missing), mark the instance offline so no jobs are dispatched to it. Previously this was silently ignored and the instance kept full capacity. 3. should_update_config(): catch FileNotFoundError and return True so write_receptor_config() can create the file on first boot instead of crashing. Prior art: #12698 (closed without merge) took a broader approach with custom exception types. See PR comment for overlap analysis. Closes: AAP-89010 Co-authored-by: Claude Opus 4.6 --- awx/main/management/commands/dispatcherd.py | 10 ++++++++ awx/main/tasks/receptor.py | 10 +++++++- awx/main/tasks/system.py | 28 ++++++++++++++------- awx/main/tests/unit/tasks/test_system.py | 20 +++++++++++++-- awx/main/tests/unit/utils/test_receptor.py | 9 ++++++- 5 files changed, 64 insertions(+), 13 deletions(-) diff --git a/awx/main/management/commands/dispatcherd.py b/awx/main/management/commands/dispatcherd.py index 69a7625571..62373d05c3 100644 --- a/awx/main/management/commands/dispatcherd.py +++ b/awx/main/management/commands/dispatcherd.py @@ -6,6 +6,7 @@ import json import logging import logging.config import os +import time from django.conf import settings from django.core.cache import cache as django_cache @@ -15,6 +16,7 @@ from django.db import connection from dispatcherd.config import setup as dispatcher_setup from awx.main.dispatch.config import get_dispatcherd_config +from awx.main.tasks.receptor import receptor_config_exists logger = logging.getLogger('awx.main.dispatch') @@ -52,6 +54,14 @@ class Command(BaseCommand): def handle(self, *arg, **options): ensure_no_dispatcherd_env_config() + for attempt in range(5): + if receptor_config_exists(): + break + logger.info("Waiting for receptor config... (attempt %d/5)", attempt + 1) + time.sleep(2) + else: + raise CommandError("Receptor config not found after 10s") + self.configure_dispatcher_logging() config = get_dispatcherd_config(for_service=True) config_hash = _hash_config(config) diff --git a/awx/main/tasks/receptor.py b/awx/main/tasks/receptor.py index dae7e49bc7..fdab631139 100644 --- a/awx/main/tasks/receptor.py +++ b/awx/main/tasks/receptor.py @@ -117,6 +117,10 @@ class FuzzyError: return UnknownError(msg, node=node, state_name=state_name) +def receptor_config_exists(): + return os.path.exists(__RECEPTOR_CONF) + + def read_receptor_config(): # for K8S deployments, getting a lock is necessary as another process # may be re-writing the config at this time @@ -801,7 +805,11 @@ def should_update_config(new_config): tcp-peers in the config ''' - current_config = read_receptor_config() # this gets receptor conf lock + try: + current_config = read_receptor_config() # this gets receptor conf lock + except FileNotFoundError: + logger.warning("Receptor config file not found, config needs to be written.") + return True for config_entry in current_config: if config_entry not in new_config: logger.warning(f"{config_entry} should not be in receptor config. Updating.") diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index be418daa0e..59486b0a8a 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -72,7 +72,13 @@ from awx.main.models import ( from awx.main.models.credential import CredentialType from awx.main.tasks.helpers import is_run_threshold_reached from awx.main.tasks.host_indirect import save_indirect_host_entries -from awx.main.tasks.receptor import administrative_workunit_reaper, get_receptor_ctl, worker_cleanup, worker_info, write_receptor_config +from awx.main.tasks.receptor import ( + administrative_workunit_reaper, + get_receptor_ctl, + worker_cleanup, + worker_info, + write_receptor_config, +) from awx.main.utils.common import ignore_inventory_computed_fields, ignore_inventory_group_removal from awx.main.utils.migration import is_database_synchronized from awx.main.utils.reload import stop_local_services @@ -582,16 +588,11 @@ def inspect_established_receptor_connections(mesh_status): InstanceLink.objects.bulk_update(update_links, ['link_state']) -def inspect_execution_and_hop_nodes(instance_list): +def inspect_execution_and_hop_nodes(instance_list, receptor_ctl): with advisory_lock('inspect_execution_and_hop_nodes_lock', wait=False): node_lookup = {inst.hostname: inst for inst in instance_list} try: - ctl = get_receptor_ctl() - except FileNotFoundError: - logger.error('Receptor daemon not running, skipping execution node check') - return - try: - mesh_status = ctl.simple_command('status') + mesh_status = receptor_ctl.simple_command('status') except ValueError as exc: logger.error(f'Error running receptorctl status command, error: {str(exc)}') return @@ -727,7 +728,16 @@ def _heartbeat_instance_management(): this_inst = inst break - inspect_execution_and_hop_nodes(instance_list) + try: + ctl = get_receptor_ctl() + except FileNotFoundError: + logger.error('Receptor not available, marking instance offline.') + if this_inst: + this_inst.local_health_check() + this_inst.mark_offline(errors='Receptor not available') + return None, None, None + + inspect_execution_and_hop_nodes(instance_list, ctl) for inst in list(instance_list): if inst == this_inst: diff --git a/awx/main/tests/unit/tasks/test_system.py b/awx/main/tests/unit/tasks/test_system.py index c567dc4833..4e18594023 100644 --- a/awx/main/tests/unit/tasks/test_system.py +++ b/awx/main/tests/unit/tasks/test_system.py @@ -1,7 +1,7 @@ import pytest from unittest.mock import MagicMock, patch -from awx.main.tasks.system import update_inventory_computed_fields -from awx.main.models import Inventory +from awx.main.tasks.system import _heartbeat_instance_management, update_inventory_computed_fields +from awx.main.models import Instance, Inventory from django.db import DatabaseError @@ -62,3 +62,19 @@ def test_update_inventory_computed_fields_database_error_nosqlstate(mock_logger, mock_filter.assert_called_once_with(id=1) mock_update_computed_fields.assert_called_once() mock_inventory.update_computed_fields.assert_called_once() + + +@patch('awx.main.tasks.system.get_receptor_ctl', side_effect=FileNotFoundError) +@patch('awx.main.tasks.system.Instance.objects.filter') +def test_heartbeat_marks_offline_when_receptor_unavailable(mock_filter, mock_get_ctl): + this_inst = MagicMock(spec=Instance) + this_inst.hostname = 'test-host' + mock_filter.return_value = [this_inst] + + with patch('awx.main.tasks.system.settings') as mock_settings: + mock_settings.CLUSTER_HOST_ID = 'test-host' + result = _heartbeat_instance_management() + + assert result == (None, None, None) + this_inst.local_health_check.assert_called_once() + this_inst.mark_offline.assert_called_once_with(errors='Receptor not available') diff --git a/awx/main/tests/unit/utils/test_receptor.py b/awx/main/tests/unit/utils/test_receptor.py index 123044fcad..4a523280a5 100644 --- a/awx/main/tests/unit/utils/test_receptor.py +++ b/awx/main/tests/unit/utils/test_receptor.py @@ -1,4 +1,6 @@ -from awx.main.tasks.receptor import _convert_args_to_cli +from unittest.mock import patch + +from awx.main.tasks.receptor import _convert_args_to_cli, should_update_config def test_file_cleanup_scenario(): @@ -20,3 +22,8 @@ def test_image_cleanup_scenario(): ' '.join(args) == 'cleanup --remove-images "quay.invalid/foo/bar:latest" "quay.invalid/foo/bar:devel" --image-prune --process-isolation-executable=podman' ) + + +def test_should_update_config_missing_file(): + with patch('awx.main.tasks.receptor.read_receptor_config', side_effect=FileNotFoundError): + assert should_update_config([{'node': {}}]) is True