From 3917f3bf748a3ac7765199795d0ffa28371bc353 Mon Sep 17 00:00:00 2001 From: Peter Braun Date: Tue, 25 Aug 2026 18:25:05 +0200 Subject: [PATCH] fix: check if the lock was acquired and return if not (#16620) * fix: check if the lock was acquired and return if not * fix linter issues --- awx/main/tasks/system.py | 14 ++++- awx/main/tests/unit/tasks/test_system.py | 65 +++++++++++++++++++++++- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index 59486b0a8a..7f83f03a3e 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -589,7 +589,11 @@ def inspect_established_receptor_connections(mesh_status): def inspect_execution_and_hop_nodes(instance_list, receptor_ctl): - with advisory_lock('inspect_execution_and_hop_nodes_lock', wait=False): + with advisory_lock('inspect_execution_and_hop_nodes_lock', wait=False) as acquired: + if not acquired: + logger.debug("Not running inspect_execution_and_hop_nodes, another instance holds lock") + return + start = time.monotonic() node_lookup = {inst.hostname: inst for inst in instance_list} try: mesh_status = receptor_ctl.simple_command('status') @@ -601,6 +605,7 @@ def inspect_execution_and_hop_nodes(instance_list, receptor_ctl): nowtime = now() workers = mesh_status['Advertisements'] + updated_count = 0 for ad in workers: hostname = ad['NodeID'] @@ -620,6 +625,7 @@ def inspect_execution_and_hop_nodes(instance_list, receptor_ctl): continue instance.last_seen = last_seen instance.save(update_fields=['last_seen']) + updated_count += 1 # Only execution nodes should be dealt with by execution_node_health_check if instance.node_type == Instance.Types.HOP: @@ -642,6 +648,12 @@ def inspect_execution_and_hop_nodes(instance_list, receptor_ctl): logger.debug(f'Restarting health check for execution node {hostname} with known errors.') execution_node_health_check.apply_async([hostname]) + elapsed = time.monotonic() - start + if elapsed > 2.0: + logger.warning(f"inspect_execution_and_hop_nodes completed in {elapsed:.1f}s, updated {updated_count} node(s)") + else: + logger.debug(f"inspect_execution_and_hop_nodes completed in {elapsed:.3f}s, updated {updated_count} node(s)") + @task(queue=get_task_queuename, bind=True) def cluster_node_heartbeat(binder): diff --git a/awx/main/tests/unit/tasks/test_system.py b/awx/main/tests/unit/tasks/test_system.py index 4e18594023..cdbf722bee 100644 --- a/awx/main/tests/unit/tasks/test_system.py +++ b/awx/main/tests/unit/tasks/test_system.py @@ -1,6 +1,6 @@ import pytest from unittest.mock import MagicMock, patch -from awx.main.tasks.system import _heartbeat_instance_management, update_inventory_computed_fields +from awx.main.tasks.system import _heartbeat_instance_management, update_inventory_computed_fields, inspect_execution_and_hop_nodes from awx.main.models import Instance, Inventory from django.db import DatabaseError @@ -78,3 +78,66 @@ def test_heartbeat_marks_offline_when_receptor_unavailable(mock_filter, mock_get assert result == (None, None, None) this_inst.local_health_check.assert_called_once() this_inst.mark_offline.assert_called_once_with(errors='Receptor not available') + + +class TestInspectExecutionAndHopNodes: + """Tests for the advisory_lock guard in inspect_execution_and_hop_nodes.""" + + @patch('awx.main.tasks.system.inspect_established_receptor_connections') + @patch('awx.main.tasks.system.advisory_lock') + def test_skips_when_lock_not_acquired(self, mock_lock, mock_inspect_conns): + mock_lock.return_value.__enter__ = MagicMock(return_value=False) + mock_lock.return_value.__exit__ = MagicMock(return_value=False) + receptor_ctl = MagicMock() + + inspect_execution_and_hop_nodes([], receptor_ctl) + + receptor_ctl.simple_command.assert_not_called() + mock_inspect_conns.assert_not_called() + + @patch('awx.main.tasks.system.inspect_established_receptor_connections') + @patch('awx.main.tasks.system.advisory_lock') + def test_runs_when_lock_acquired(self, mock_lock, mock_inspect_conns): + mock_lock.return_value.__enter__ = MagicMock(return_value=True) + mock_lock.return_value.__exit__ = MagicMock(return_value=False) + receptor_ctl = MagicMock() + receptor_ctl.simple_command.return_value = {'Advertisements': [], 'KnownConnectionCosts': {}} + + inspect_execution_and_hop_nodes([], receptor_ctl) + + receptor_ctl.simple_command.assert_called_once_with('status') + mock_inspect_conns.assert_called_once() + + @patch('awx.main.tasks.system.inspect_established_receptor_connections') + @patch('awx.main.tasks.system.advisory_lock') + def test_updates_last_seen_for_execution_nodes(self, mock_lock, mock_inspect_conns): + mock_lock.return_value.__enter__ = MagicMock(return_value=True) + mock_lock.return_value.__exit__ = MagicMock(return_value=False) + + exec_node = MagicMock(spec=Instance) + exec_node.hostname = 'exec-1' + exec_node.node_type = 'execution' + exec_node.node_state = 'ready' + exec_node.last_seen = None + exec_node.capacity = 100 + exec_node.enabled = True + exec_node.cpu = 4 + exec_node.memory = 8000000000 + + control_node = MagicMock(spec=Instance) + control_node.hostname = 'control-1' + control_node.node_type = 'control' + + receptor_ctl = MagicMock() + receptor_ctl.simple_command.return_value = { + 'Advertisements': [ + {'NodeID': 'exec-1', 'Time': '2026-01-01T00:00:00+00:00'}, + {'NodeID': 'control-1', 'Time': '2026-01-01T00:00:00+00:00'}, + ], + 'KnownConnectionCosts': {}, + } + + inspect_execution_and_hop_nodes([exec_node, control_node], receptor_ctl) + + exec_node.save.assert_called_once_with(update_fields=['last_seen']) + control_node.save.assert_not_called()