diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index 362ff45c29..fc2a04fa93 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -733,6 +733,9 @@ def _mesh_all_ready_nodes_visible(mesh_status): populates this table as connections establish via gossip. An empty table means no routing has propagated yet (Window A — typically the first ~10s after restart). + In Kubernetes (IS_K8S=True), controller pods are stateless and independent with no + peer connections expected. Empty routing is the normal steady state, not instability. + No DB state is consulted. KnownConnectionCosts is maintained entirely by the receptor Go process, making it a reliable mesh-state signal free of stale DB records. @@ -741,6 +744,8 @@ def _mesh_all_ready_nodes_visible(mesh_status): """ if mesh_status is None: return True # fail open: status unavailable, let normal peer-judgment proceed + if settings.IS_K8S: + return True # K8s pods are stateless; no mesh consensus required if not (mesh_status.get('KnownConnectionCosts') or {}): logger.info('Mesh stability gate: routing table empty, deferring peer-judgment (receptor re-establishing)') return False @@ -806,7 +811,10 @@ def _heartbeat_instance_management(): return None, None, None if lost_instances and not _mesh_all_ready_nodes_visible(mesh_status): - return this_inst, instance_list, [] + # Mesh gate blocks cleanup, but execution and hop nodes can still be reaped + # (they don't depend on mesh consensus). Defer only control nodes. + execution_hop_lost = [inst for inst in lost_instances if inst.node_type in ('execution', 'hop')] + return this_inst, instance_list, execution_hop_lost return this_inst, instance_list, lost_instances diff --git a/awx/main/tests/functional/tasks/test_tasks_system.py b/awx/main/tests/functional/tasks/test_tasks_system.py index edd3178305..66da63cac7 100644 --- a/awx/main/tests/functional/tasks/test_tasks_system.py +++ b/awx/main/tests/functional/tasks/test_tasks_system.py @@ -548,6 +548,105 @@ def test_heartbeat_defers_lost_instances_when_mesh_gate_blocks(settings): assert lost_result == [] +@pytest.mark.parametrize( + 'mesh_status', + [ + {'KnownConnectionCosts': {}}, + {'Connections': [], 'KnownConnectionCosts': {}, 'RoutingTable': {}}, + ], +) +@pytest.mark.django_db +def test_kubernetes_mesh_gate_allows_empty_routing(settings, mesh_status): + """In Kubernetes (IS_K8S=True), empty routing is normal — gate passes.""" + settings.IS_K8S = True + assert _mesh_all_ready_nodes_visible(mesh_status) is True + + +@pytest.mark.django_db +def test_non_kubernetes_mesh_gate_blocks_empty_routing(settings): + """Non-Kubernetes deployments block cleanup when routing is empty.""" + settings.IS_K8S = False + assert _mesh_all_ready_nodes_visible({'KnownConnectionCosts': {}}) is False + + +@pytest.mark.django_db +def test_kubernetes_passes_mesh_gate_with_empty_routing(settings): + """In Kubernetes (IS_K8S=True), empty routing is normal — gate passes.""" + settings.CLUSTER_HOST_ID = 'ctrl-0' + settings.AWX_AUTO_DEPROVISION_INSTANCES = False + settings.CLUSTER_NODE_HEARTBEAT_PERIOD = 60 + settings.CLUSTER_NODE_MISSED_HEARTBEAT_TOLERANCE = 2 + settings.IS_K8S = True + + this_inst = Instance.objects.create(hostname='ctrl-0', node_type='control', node_state='ready') + this_inst.last_seen = now() - timedelta(seconds=30) + this_inst.save(update_fields=['last_seen']) + + lost_inst = Instance.objects.create(hostname='ctrl-1', node_type='control', node_state='ready') + lost_inst.last_seen = now() - timedelta(seconds=200) + lost_inst.save(update_fields=['last_seen']) + + mock_ctl = mock.MagicMock() + # Empty routing in K8s is expected steady state + mock_ctl.simple_command.return_value = {'KnownConnectionCosts': {}, 'Advertisements': []} + + with ( + mock.patch('awx.main.tasks.system.get_receptor_ctl', return_value=mock_ctl), + mock.patch('awx.main.tasks.system.inspect_execution_and_hop_nodes'), + mock.patch.object(Instance, 'local_health_check'), + ): + _, _, lost_result = _heartbeat_instance_management() + + # K8s bypasses the gate — lost instances are reaped even with empty routing + assert len(lost_result) == 1 + assert lost_result[0].hostname == 'ctrl-1' + + +@pytest.mark.django_db +def test_mesh_gate_defers_control_but_cleans_execution_hop(settings): + """When mesh gate blocks cleanup, execution/hop nodes are reaped but control nodes are deferred.""" + settings.CLUSTER_HOST_ID = 'ctrl-0' + settings.AWX_AUTO_DEPROVISION_INSTANCES = False + settings.CLUSTER_NODE_HEARTBEAT_PERIOD = 60 + settings.CLUSTER_NODE_MISSED_HEARTBEAT_TOLERANCE = 2 + settings.IS_K8S = False # Non-K8s: mesh gate can block + + this_inst = Instance.objects.create(hostname='ctrl-0', node_type='control', node_state='ready') + this_inst.last_seen = now() - timedelta(seconds=30) + this_inst.save(update_fields=['last_seen']) + + # Create lost instances of different types + lost_exec = Instance.objects.create(hostname='exec-1', node_type='execution', node_state='ready') + lost_exec.last_seen = now() - timedelta(seconds=200) + lost_exec.save(update_fields=['last_seen']) + + lost_hop = Instance.objects.create(hostname='hop-1', node_type='hop', node_state='ready') + lost_hop.last_seen = now() - timedelta(seconds=200) + lost_hop.save(update_fields=['last_seen']) + + lost_ctrl = Instance.objects.create(hostname='ctrl-1', node_type='control', node_state='ready') + lost_ctrl.last_seen = now() - timedelta(seconds=200) + lost_ctrl.save(update_fields=['last_seen']) + + mock_ctl = mock.MagicMock() + # Empty routing — mesh gate will block + mock_ctl.simple_command.return_value = {'KnownConnectionCosts': {}, 'Advertisements': []} + + with ( + mock.patch('awx.main.tasks.system.get_receptor_ctl', return_value=mock_ctl), + mock.patch('awx.main.tasks.system.inspect_execution_and_hop_nodes'), + mock.patch.object(Instance, 'local_health_check'), + ): + _, _, lost_result = _heartbeat_instance_management() + + # Execution and hop nodes should be in lost_result (reaped immediately) + # Control node should be deferred (not in lost_result) + hostnames = [inst.hostname for inst in lost_result] + assert 'exec-1' in hostnames, 'Execution node should be in lost list even when mesh gate blocks' + assert 'hop-1' in hostnames, 'Hop node should be in lost list even when mesh gate blocks' + assert 'ctrl-1' not in hostnames, 'Control node should be deferred when mesh gate blocks' + + @pytest.mark.django_db def test_heartbeat_marks_offline_when_receptor_unavailable(settings): """FileNotFoundError from get_receptor_ctl → this_inst marked offline, returns (None, None, None)."""