From 549999850651d2e52373b63d699eb75a49f1266a Mon Sep 17 00:00:00 2001 From: Hui Song Date: Thu, 17 Sep 2026 10:29:28 -0400 Subject: [PATCH] fix: [AAP-92631] Only defer control nodes in mesh stability gate (#16655) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: AAP-92631 Only defer control nodes in mesh stability gate When the mesh routing table is empty, the stability gate was blocking cleanup of ALL lost instances (both control and execution nodes). However, execution nodes don't need the mesh and can be safely reaped immediately. This fix separates lost instances by node type: - Control nodes: defer cleanup (they need mesh consensus) - Execution/hop nodes: reap immediately (they don't need mesh) The previous behavior caused stale execution node instances to accumulate in the database indefinitely, preventing job cleanup and degrading controller performance. Fixes: AAP-92631 Co-Authored-By: Claude Haiku 4.5 * fix: AAP-92631 Handle Kubernetes mesh gate differently from cluster deployments The mesh stability gate checks KnownConnectionCosts to detect if receptor routing is established. In cluster/mesh deployments, empty routing means the mesh is re-establishing after a controller restart, so the gate blocks cleanup of lost instances to prevent cluster fragmentation. However, in Kubernetes (IS_K8S=True), controller pods are stateless and independent — they don't need peer connections. Empty routing is the expected steady state, not a sign of instability. The gate was treating both deployments identically, causing lost instance rows to accumulate indefinitely in Kubernetes. This fix adds an IS_K8S check: when Kubernetes mode is enabled, the gate always passes (returns True), allowing normal cleanup. In cluster mode, the gate continues to check KnownConnectionCosts. Fixes: AAP-92631 Co-Authored-By: Claude Haiku 4.5 * fix: [AAP-92631] Return execution/hop nodes when mesh gate blocks cleanup ## Issue When the non-K8s mesh gate blocked cleanup (empty routing table), the code returned an empty list, preventing execution and hop node cleanup. This was unintended—these nodes don't depend on mesh consensus and should be reaped immediately. ## Fix - Filter lost_instances to return only execution/hop nodes when mesh gate blocks - Control nodes (which require mesh consensus) continue to be deferred - K8s deployments bypass the gate entirely (empty routing is steady state) ## Test Coverage Added three tests per reviewer request: 1. test_kubernetes_mesh_gate_allows_empty_routing (parametrized) - Unit test verifying gate logic with different mesh_status shapes - Confirms _mesh_all_ready_nodes_visible() returns True in K8s 2. test_non_kubernetes_mesh_gate_blocks_empty_routing - Unit test confirming non-K8s gate blocks with empty routing 3. test_mesh_gate_defers_control_but_cleans_execution_hop - E2E integration test with all three node types - Verifies exec/hop nodes reach cleanup while control nodes defer - Matches production failure scenario from baremetal-a cluster All 18 heartbeat/mesh tests pass. Co-Authored-By: Claude Haiku 4.5 --------- Co-authored-by: Claude Haiku 4.5 --- awx/main/tasks/system.py | 10 +- .../functional/tasks/test_tasks_system.py | 99 +++++++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index 362ff45c29..fc2a04fa93 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -733,6 +733,9 @@ def _mesh_all_ready_nodes_visible(mesh_status): populates this table as connections establish via gossip. An empty table means no routing has propagated yet (Window A — typically the first ~10s after restart). + In Kubernetes (IS_K8S=True), controller pods are stateless and independent with no + peer connections expected. Empty routing is the normal steady state, not instability. + No DB state is consulted. KnownConnectionCosts is maintained entirely by the receptor Go process, making it a reliable mesh-state signal free of stale DB records. @@ -741,6 +744,8 @@ def _mesh_all_ready_nodes_visible(mesh_status): """ if mesh_status is None: return True # fail open: status unavailable, let normal peer-judgment proceed + if settings.IS_K8S: + return True # K8s pods are stateless; no mesh consensus required if not (mesh_status.get('KnownConnectionCosts') or {}): logger.info('Mesh stability gate: routing table empty, deferring peer-judgment (receptor re-establishing)') return False @@ -806,7 +811,10 @@ def _heartbeat_instance_management(): return None, None, None if lost_instances and not _mesh_all_ready_nodes_visible(mesh_status): - return this_inst, instance_list, [] + # Mesh gate blocks cleanup, but execution and hop nodes can still be reaped + # (they don't depend on mesh consensus). Defer only control nodes. + execution_hop_lost = [inst for inst in lost_instances if inst.node_type in ('execution', 'hop')] + return this_inst, instance_list, execution_hop_lost return this_inst, instance_list, lost_instances diff --git a/awx/main/tests/functional/tasks/test_tasks_system.py b/awx/main/tests/functional/tasks/test_tasks_system.py index edd3178305..66da63cac7 100644 --- a/awx/main/tests/functional/tasks/test_tasks_system.py +++ b/awx/main/tests/functional/tasks/test_tasks_system.py @@ -548,6 +548,105 @@ def test_heartbeat_defers_lost_instances_when_mesh_gate_blocks(settings): assert lost_result == [] +@pytest.mark.parametrize( + 'mesh_status', + [ + {'KnownConnectionCosts': {}}, + {'Connections': [], 'KnownConnectionCosts': {}, 'RoutingTable': {}}, + ], +) +@pytest.mark.django_db +def test_kubernetes_mesh_gate_allows_empty_routing(settings, mesh_status): + """In Kubernetes (IS_K8S=True), empty routing is normal — gate passes.""" + settings.IS_K8S = True + assert _mesh_all_ready_nodes_visible(mesh_status) is True + + +@pytest.mark.django_db +def test_non_kubernetes_mesh_gate_blocks_empty_routing(settings): + """Non-Kubernetes deployments block cleanup when routing is empty.""" + settings.IS_K8S = False + assert _mesh_all_ready_nodes_visible({'KnownConnectionCosts': {}}) is False + + +@pytest.mark.django_db +def test_kubernetes_passes_mesh_gate_with_empty_routing(settings): + """In Kubernetes (IS_K8S=True), empty routing is normal — gate passes.""" + settings.CLUSTER_HOST_ID = 'ctrl-0' + settings.AWX_AUTO_DEPROVISION_INSTANCES = False + settings.CLUSTER_NODE_HEARTBEAT_PERIOD = 60 + settings.CLUSTER_NODE_MISSED_HEARTBEAT_TOLERANCE = 2 + settings.IS_K8S = True + + this_inst = Instance.objects.create(hostname='ctrl-0', node_type='control', node_state='ready') + this_inst.last_seen = now() - timedelta(seconds=30) + this_inst.save(update_fields=['last_seen']) + + lost_inst = Instance.objects.create(hostname='ctrl-1', node_type='control', node_state='ready') + lost_inst.last_seen = now() - timedelta(seconds=200) + lost_inst.save(update_fields=['last_seen']) + + mock_ctl = mock.MagicMock() + # Empty routing in K8s is expected steady state + mock_ctl.simple_command.return_value = {'KnownConnectionCosts': {}, 'Advertisements': []} + + with ( + mock.patch('awx.main.tasks.system.get_receptor_ctl', return_value=mock_ctl), + mock.patch('awx.main.tasks.system.inspect_execution_and_hop_nodes'), + mock.patch.object(Instance, 'local_health_check'), + ): + _, _, lost_result = _heartbeat_instance_management() + + # K8s bypasses the gate — lost instances are reaped even with empty routing + assert len(lost_result) == 1 + assert lost_result[0].hostname == 'ctrl-1' + + +@pytest.mark.django_db +def test_mesh_gate_defers_control_but_cleans_execution_hop(settings): + """When mesh gate blocks cleanup, execution/hop nodes are reaped but control nodes are deferred.""" + settings.CLUSTER_HOST_ID = 'ctrl-0' + settings.AWX_AUTO_DEPROVISION_INSTANCES = False + settings.CLUSTER_NODE_HEARTBEAT_PERIOD = 60 + settings.CLUSTER_NODE_MISSED_HEARTBEAT_TOLERANCE = 2 + settings.IS_K8S = False # Non-K8s: mesh gate can block + + this_inst = Instance.objects.create(hostname='ctrl-0', node_type='control', node_state='ready') + this_inst.last_seen = now() - timedelta(seconds=30) + this_inst.save(update_fields=['last_seen']) + + # Create lost instances of different types + lost_exec = Instance.objects.create(hostname='exec-1', node_type='execution', node_state='ready') + lost_exec.last_seen = now() - timedelta(seconds=200) + lost_exec.save(update_fields=['last_seen']) + + lost_hop = Instance.objects.create(hostname='hop-1', node_type='hop', node_state='ready') + lost_hop.last_seen = now() - timedelta(seconds=200) + lost_hop.save(update_fields=['last_seen']) + + lost_ctrl = Instance.objects.create(hostname='ctrl-1', node_type='control', node_state='ready') + lost_ctrl.last_seen = now() - timedelta(seconds=200) + lost_ctrl.save(update_fields=['last_seen']) + + mock_ctl = mock.MagicMock() + # Empty routing — mesh gate will block + mock_ctl.simple_command.return_value = {'KnownConnectionCosts': {}, 'Advertisements': []} + + with ( + mock.patch('awx.main.tasks.system.get_receptor_ctl', return_value=mock_ctl), + mock.patch('awx.main.tasks.system.inspect_execution_and_hop_nodes'), + mock.patch.object(Instance, 'local_health_check'), + ): + _, _, lost_result = _heartbeat_instance_management() + + # Execution and hop nodes should be in lost_result (reaped immediately) + # Control node should be deferred (not in lost_result) + hostnames = [inst.hostname for inst in lost_result] + assert 'exec-1' in hostnames, 'Execution node should be in lost list even when mesh gate blocks' + assert 'hop-1' in hostnames, 'Hop node should be in lost list even when mesh gate blocks' + assert 'ctrl-1' not in hostnames, 'Control node should be deferred when mesh gate blocks' + + @pytest.mark.django_db def test_heartbeat_marks_offline_when_receptor_unavailable(settings): """FileNotFoundError from get_receptor_ctl → this_inst marked offline, returns (None, None, None)."""