diff --git a/awx/main/tasks/callback.py b/awx/main/tasks/callback.py index c6d89d0b79..266fd22015 100644 --- a/awx/main/tasks/callback.py +++ b/awx/main/tasks/callback.py @@ -54,9 +54,6 @@ def try_load_query_file(artifact_dir) -> Tuple[bool, Optional[dict]]: returns the contents of ansible_data.json if present """ - if not flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"): - return False, None - queries_path = os.path.join(artifact_dir, COLLECTION_FILENAME) if not os.path.isfile(queries_path): logger.info(f"no query file found: {queries_path}") @@ -277,20 +274,6 @@ class RunnerCallback: def artifacts_handler(self, artifact_dir): success, query_file_contents = try_load_query_file(artifact_dir) if success: - self.delay_update(event_queries_processed=False) - collections_info = collect_queries(query_file_contents) - for collection, data in collections_info.items(): - version = data['version'] - event_query = data['host_query'] - instance = EventQuery(fqcn=collection, collection_version=version, event_query=event_query) - try: - instance.validate_unique() - instance.save() - - logger.info(f"eventy query for collection {collection}, version {version} created") - except ValidationError as e: - logger.info(e) - if 'installed_collections' in query_file_contents: self.delay_update(installed_collections=query_file_contents['installed_collections']) else: @@ -301,6 +284,21 @@ class RunnerCallback: else: logger.warning(f'The file {COLLECTION_FILENAME} unexpectedly did not contain ansible_version') + if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"): + self.delay_update(event_queries_processed=False) + collections_info = collect_queries(query_file_contents) + for collection, data in collections_info.items(): + version = data['version'] + event_query = data['host_query'] + instance = EventQuery(fqcn=collection, collection_version=version, event_query=event_query) + try: + instance.validate_unique() + instance.save() + + logger.info(f"event query for collection {collection}, version {version} created") + except ValidationError as e: + logger.info(e) + self.artifacts_processed = True diff --git a/awx/main/tasks/jobs.py b/awx/main/tasks/jobs.py index 69250e967d..49d820ffeb 100644 --- a/awx/main/tasks/jobs.py +++ b/awx/main/tasks/jobs.py @@ -1138,10 +1138,9 @@ class RunJob(SourceControlMixin, BaseTask): ('ANSIBLE_COLLECTIONS_PATH', 'collections_path', 'requirements_collections', '~/.ansible/collections:/usr/share/ansible/collections'), ] - if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"): - path_vars.append( - ('ANSIBLE_CALLBACK_PLUGINS', 'callback_plugins', 'plugins_path', '~/.ansible/plugins:/plugins/callback:/usr/share/ansible/plugins/callback'), - ) + path_vars.append( + ('ANSIBLE_CALLBACK_PLUGINS', 'callback_plugins', 'plugins_path', '~/.ansible/plugins:/plugins/callback:/usr/share/ansible/plugins/callback'), + ) config_values = read_ansible_config(os.path.join(private_data_dir, 'project'), list(map(lambda x: x[1], path_vars))) @@ -1158,11 +1157,11 @@ class RunJob(SourceControlMixin, BaseTask): paths = [os.path.join(CONTAINER_ROOT, folder)] + paths env[env_key] = os.pathsep.join(paths) - if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"): - env['ANSIBLE_CALLBACKS_ENABLED'] = 'indirect_instance_count' - if 'callbacks_enabled' in config_values: - env['ANSIBLE_CALLBACKS_ENABLED'] += ':' + config_values['callbacks_enabled'] + env['ANSIBLE_CALLBACKS_ENABLED'] = 'indirect_instance_count' + if 'callbacks_enabled' in config_values: + env['ANSIBLE_CALLBACKS_ENABLED'] += ':' + config_values['callbacks_enabled'] + if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"): # Add vendor collections path for external query file discovery vendor_collections_path = os.path.join(CONTAINER_ROOT, 'vendor_collections') env['ANSIBLE_COLLECTIONS_PATH'] = f"{vendor_collections_path}:{env['ANSIBLE_COLLECTIONS_PATH']}" @@ -1612,16 +1611,14 @@ class RunProjectUpdate(BaseTask): shutil.copytree(cache_subpath, dest_subpath, symlinks=True) logger.debug('{0} {1} prepared {2} from cache'.format(type(project).__name__, project.pk, dest_subpath)) - if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"): - # copy the special callback (not stdout type) plugin to get list of collections - pdd_plugins_path = os.path.join(job_private_data_dir, 'plugins_path') - if not os.path.exists(pdd_plugins_path): - os.mkdir(pdd_plugins_path) - from awx.playbooks import library + pdd_plugins_path = os.path.join(job_private_data_dir, 'plugins_path') + if not os.path.exists(pdd_plugins_path): + os.mkdir(pdd_plugins_path) + from awx.playbooks import library - plugin_file_source = os.path.join(library.__path__._path[0], 'indirect_instance_count.py') - plugin_file_dest = os.path.join(pdd_plugins_path, 'indirect_instance_count.py') - shutil.copyfile(plugin_file_source, plugin_file_dest) + plugin_file_source = os.path.join(library.__path__._path[0], 'indirect_instance_count.py') + plugin_file_dest = os.path.join(pdd_plugins_path, 'indirect_instance_count.py') + shutil.copyfile(plugin_file_source, plugin_file_dest) def post_run_hook(self, instance, status): super(RunProjectUpdate, self).post_run_hook(instance, status) diff --git a/awx/main/tests/unit/tasks/test_runner_callback.py b/awx/main/tests/unit/tasks/test_runner_callback.py index fb04842e10..54c964cc1c 100644 --- a/awx/main/tests/unit/tasks/test_runner_callback.py +++ b/awx/main/tests/unit/tasks/test_runner_callback.py @@ -1,4 +1,9 @@ -from awx.main.tasks.callback import RunnerCallback +import json +import os +import tempfile +from unittest import mock + +from awx.main.tasks.callback import RunnerCallback, try_load_query_file from awx.main.constants import ANSIBLE_RUNNER_NEEDS_UPDATE_MESSAGE from django.utils.translation import gettext_lazy as _ @@ -50,3 +55,102 @@ def test_special_ansible_runner_message(mock_me): 'Traceback:\ngot an unexpected keyword argument\nFile: bar.py\n' f'{ANSIBLE_RUNNER_NEEDS_UPDATE_MESSAGE}' ) + + +SAMPLE_ANSIBLE_DATA = { + 'installed_collections': { + 'ansible.builtin': {'version': '2.16.0'}, + 'community.general': {'version': '8.0.0', 'host_query': 'SELECT * FROM hosts'}, + }, + 'ansible_version': '2.16.0', +} + + +class TestTryLoadQueryFile: + def test_loads_file_without_feature_flag(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, 'ansible_data.json') + with open(path, 'w') as f: + json.dump(SAMPLE_ANSIBLE_DATA, f) + + with mock.patch('awx.main.tasks.callback.flag_enabled', return_value=False): + success, data = try_load_query_file(tmpdir) + + assert success is True + assert data['ansible_version'] == '2.16.0' + assert 'ansible.builtin' in data['installed_collections'] + + def test_loads_file_with_feature_flag(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, 'ansible_data.json') + with open(path, 'w') as f: + json.dump(SAMPLE_ANSIBLE_DATA, f) + + with mock.patch('awx.main.tasks.callback.flag_enabled', return_value=True): + success, data = try_load_query_file(tmpdir) + + assert success is True + assert data == SAMPLE_ANSIBLE_DATA + + def test_returns_false_when_file_missing(self): + with tempfile.TemporaryDirectory() as tmpdir: + success, data = try_load_query_file(tmpdir) + + assert success is False + assert data is None + + +class TestArtifactsHandler: + def test_always_persists_metadata_when_flag_off(self, mock_me): + rc = RunnerCallback() + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, 'ansible_data.json') + with open(path, 'w') as f: + json.dump(SAMPLE_ANSIBLE_DATA, f) + + with mock.patch('awx.main.tasks.callback.flag_enabled', return_value=False): + rc.artifacts_handler(tmpdir) + + assert rc.extra_update_fields['installed_collections'] == SAMPLE_ANSIBLE_DATA['installed_collections'] + assert rc.extra_update_fields['ansible_version'] == '2.16.0' + assert 'event_queries_processed' not in rc.extra_update_fields + assert rc.artifacts_processed is True + + @mock.patch('awx.main.tasks.callback.EventQuery') + def test_creates_event_queries_when_flag_on(self, mock_event_query, mock_me): + rc = RunnerCallback() + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, 'ansible_data.json') + with open(path, 'w') as f: + json.dump(SAMPLE_ANSIBLE_DATA, f) + + with mock.patch('awx.main.tasks.callback.flag_enabled', return_value=True): + rc.artifacts_handler(tmpdir) + + assert rc.extra_update_fields['installed_collections'] == SAMPLE_ANSIBLE_DATA['installed_collections'] + assert rc.extra_update_fields['ansible_version'] == '2.16.0' + assert rc.extra_update_fields['event_queries_processed'] is False + mock_event_query.assert_called_once() + + @mock.patch('awx.main.tasks.callback.EventQuery') + def test_no_event_queries_when_flag_off(self, mock_event_query, mock_me): + rc = RunnerCallback() + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, 'ansible_data.json') + with open(path, 'w') as f: + json.dump(SAMPLE_ANSIBLE_DATA, f) + + with mock.patch('awx.main.tasks.callback.flag_enabled', return_value=False): + rc.artifacts_handler(tmpdir) + + mock_event_query.assert_not_called() + + def test_handles_missing_artifact_file(self, mock_me): + rc = RunnerCallback() + with tempfile.TemporaryDirectory() as tmpdir: + with mock.patch('awx.main.tasks.callback.flag_enabled', return_value=False): + rc.artifacts_handler(tmpdir) + + assert 'installed_collections' not in rc.extra_update_fields + assert 'ansible_version' not in rc.extra_update_fields + assert rc.artifacts_processed is True