diff --git a/awx/main/tasks/host_indirect.py b/awx/main/tasks/host_indirect.py index 26a75f1850..d655b9f8a5 100644 --- a/awx/main/tasks/host_indirect.py +++ b/awx/main/tasks/host_indirect.py @@ -45,22 +45,35 @@ def build_indirect_host_data(job: Job, job_event_queries: dict[str, dict[str, st facts_missing_logged = False unhashable_facts_logged = False + job_event_queries_fqcn = {} + for query_k, query_v in job_event_queries.items(): + if len(parts := query_k.split('.')) != 3: + logger.info(f"Skiping malformed query '{query_k}'. Expected to be of the form 'a.b.c'") + continue + if parts[2] != '*': + continue + job_event_queries_fqcn['.'.join(parts[0:2])] = query_v + for event in job.job_events.filter(event_data__isnull=False).iterator(): if 'res' not in event.event_data: continue - if 'resolved_action' not in event.event_data or event.event_data['resolved_action'] not in job_event_queries.keys(): + if not (resolved_action := event.event_data.get('resolved_action', None)): continue - resolved_action = event.event_data['resolved_action'] + if len(resolved_action_parts := resolved_action.split('.')) != 3: + logger.debug(f"Malformed invocation module name '{resolved_action}'. Expected to be of the form 'a.b.c'") + continue - # We expect a dict with a 'query' key for the resolved_action - if 'query' not in job_event_queries[resolved_action]: + resolved_action_fqcn = '.'.join(resolved_action_parts[0:2]) + + # Match module invocation to collection queries + # First match against fully qualified query names i.e. a.b.c + # Then try and match against wildcard queries i.e. a.b.* + if not (jq_str_for_event := job_event_queries.get(resolved_action, job_event_queries_fqcn.get(resolved_action_fqcn, {})).get('query')): continue # Recall from cache, or process the jq expression, and loop over the jq results - jq_str_for_event = job_event_queries[resolved_action]['query'] - if jq_str_for_event not in compiled_jq_expressions: compiled_jq_expressions[resolved_action] = jq.compile(jq_str_for_event) compiled_jq = compiled_jq_expressions[resolved_action] diff --git a/awx/main/tests/functional/tasks/test_host_indirect.py b/awx/main/tests/functional/tasks/test_host_indirect.py index 51cc5e8241..cfa98d2391 100644 --- a/awx/main/tests/functional/tasks/test_host_indirect.py +++ b/awx/main/tests/functional/tasks/test_host_indirect.py @@ -1,4 +1,5 @@ import yaml +from functools import reduce from unittest import mock import pytest @@ -20,6 +21,46 @@ from awx.main.models.indirect_managed_node_audit import IndirectManagedNodeAudit TEST_JQ = "{name: .name, canonical_facts: {host_name: .direct_host_name}, facts: {another_host_name: .direct_host_name}}" +class Query(dict): + def __init__(self, resolved_action: str, query_jq: dict): + self._resolved_action = resolved_action.split('.') + self._collection_ns, self._collection_name, self._module_name = self._resolved_action + + super().__init__({self.resolve_key: {'query': query_jq}}) + + def get_fqcn(self): + return f'{self._collection_ns}.{self._collection_name}' + + @property + def resolve_value(self): + return self[self.resolve_key] + + @property + def resolve_key(self): + return f'{self.get_fqcn()}.{self._module_name}' + + def resolve(self, module_name=None): + return {f'{self.get_fqcn()}.{module_name or self._module_name}': self.resolve_value} + + def create_event_query(self, module_name=None): + if (module_name := module_name or self._module_name) == '*': + raise ValueError('Invalid module name *') + return self.create_event_queries([module_name]) + + def create_event_queries(self, module_names): + queries = {} + for name in module_names: + queries |= self.resolve(name) + return EventQuery.objects.create( + fqcn=self.get_fqcn(), + collection_version='1.0.1', + event_query=yaml.dump(queries, default_flow_style=False), + ) + + def create_registered_event(self, job, module_name): + job.job_events.create(event_data={'resolved_action': f'{self.get_fqcn()}.{module_name}', 'res': {'direct_host_name': 'foo_host', 'name': 'vm-foo'}}) + + @pytest.fixture def bare_job(job_factory): job = job_factory() @@ -39,11 +80,6 @@ def job_with_counted_event(bare_job): return bare_job -def create_event_query(fqcn='demo.query'): - module_name = f'{fqcn}.example' - return EventQuery.objects.create(fqcn=fqcn, collection_version='1.0.1', event_query=yaml.dump({module_name: {'query': TEST_JQ}}, default_flow_style=False)) - - def create_audit_record(name, job, organization, created=now()): record = IndirectManagedNodeAudit.objects.create(name=name, job=job, organization=organization) record.created = created @@ -54,7 +90,7 @@ def create_audit_record(name, job, organization, created=now()): @pytest.fixture def event_query(): "This is ordinarily created by the artifacts callback" - return create_event_query() + return Query('demo.query.example', TEST_JQ).create_event_query() @pytest.fixture @@ -72,105 +108,211 @@ def new_audit_record(bare_job, organization): @pytest.mark.django_db -def test_build_with_no_results(bare_job): - # never filled in events, should do nothing - assert build_indirect_host_data(bare_job, {}) == [] +@pytest.mark.parametrize( + 'queries,expected_matches', + ( + pytest.param( + [], + 0, + id='no_results', + ), + pytest.param( + [Query('demo.query.example', TEST_JQ)], + 1, + id='fully_qualified', + ), + pytest.param( + [Query('demo.query.*', TEST_JQ)], + 1, + id='wildcard', + ), + pytest.param( + [ + Query('demo.query.*', TEST_JQ), + Query('demo.query.example', TEST_JQ), + ], + 1, + id='wildcard_and_fully_qualified', + ), + pytest.param( + [ + Query('demo.query.*', TEST_JQ), + Query('demo.query.example', {}), + ], + 0, + id='wildcard_and_fully_qualified', + ), + pytest.param( + [ + Query('demo.query.example', {}), + Query('demo.query.*', TEST_JQ), + ], + 0, + id='ordering_should_not_matter', + ), + ), +) +def test_build_indirect_host_data(job_with_counted_event, queries: Query, expected_matches: int): + data = build_indirect_host_data(job_with_counted_event, {k: v for d in queries for k, v in d.items()}) + assert len(data) == expected_matches + + +@mock.patch('awx.main.tasks.host_indirect.logger.debug') +@pytest.mark.django_db +@pytest.mark.parametrize( + 'task_name', + ( + pytest.param( + 'demo.query', + id='no_results', + ), + pytest.param( + 'demo', + id='no_results', + ), + pytest.param( + 'a.b.c.d', + id='no_results', + ), + ), +) +def test_build_indirect_host_data_malformed_module_name(mock_logger_debug, bare_job, task_name: str): + create_registered_event(bare_job, task_name) + assert build_indirect_host_data(bare_job, Query('demo.query.example', TEST_JQ)) == [] + mock_logger_debug.assert_called_once_with(f"Malformed invocation module name '{task_name}'. Expected to be of the form 'a.b.c'") + + +@mock.patch('awx.main.tasks.host_indirect.logger.info') +@pytest.mark.django_db +@pytest.mark.parametrize( + 'query', + ( + pytest.param( + 'demo.query', + id='no_results', + ), + pytest.param( + 'demo', + id='no_results', + ), + pytest.param( + 'a.b.c.d', + id='no_results', + ), + ), +) +def test_build_indirect_host_data_malformed_query(mock_logger_info, job_with_counted_event, query: str): + assert build_indirect_host_data(job_with_counted_event, {query: {'query': TEST_JQ}}) == [] + mock_logger_info.assert_called_once_with(f"Skiping malformed query '{query}'. Expected to be of the form 'a.b.c'") @pytest.mark.django_db -def test_collect_an_event(job_with_counted_event): - records = build_indirect_host_data(job_with_counted_event, {'demo.query.example': {'query': TEST_JQ}}) - assert len(records) == 1 +@pytest.mark.parametrize( + 'query', + ( + pytest.param( + Query('demo.query.example', TEST_JQ), + id='fully_qualified', + ), + pytest.param( + Query('demo.query.*', TEST_JQ), + id='wildcard', + ), + ), +) +def test_fetch_job_event_query(bare_job, query: Query): + query.create_event_query(module_name='example') + assert fetch_job_event_query(bare_job) == query.resolve('example') @pytest.mark.django_db -def test_fetch_job_event_query(bare_job, event_query): - assert fetch_job_event_query(bare_job) == {'demo.query.example': {'query': TEST_JQ}} +@pytest.mark.parametrize( + 'queries', + ( + [ + Query('demo.query.example', TEST_JQ), + Query('demo2.query.example', TEST_JQ), + ], + [ + Query('demo.query.*', TEST_JQ), + Query('demo2.query.example', TEST_JQ), + ], + ), +) +def test_fetch_multiple_job_event_query(bare_job, queries: list[Query]): + for q in queries: + q.create_event_query(module_name='example') + assert fetch_job_event_query(bare_job) == reduce(lambda acc, q: acc | q.resolve('example'), queries, {}) @pytest.mark.django_db -def test_fetch_multiple_job_event_query(bare_job): - create_event_query(fqcn='demo.query') - create_event_query(fqcn='demo2.query') - assert fetch_job_event_query(bare_job) == {'demo.query.example': {'query': TEST_JQ}, 'demo2.query.example': {'query': TEST_JQ}} +@pytest.mark.parametrize( + ('state',), + ( + pytest.param( + [ + ( + Query('demo.query.example', TEST_JQ), + ['example'], + ), + ], + id='fully_qualified', + ), + pytest.param( + [ + ( + Query('demo.query.example', TEST_JQ), + ['example'] * 3, + ), + ], + id='multiple_events_same_module_same_host', + ), + pytest.param( + [ + ( + Query('demo.query.example', TEST_JQ), + ['example'], + ), + ( + Query('demo2.query.example', TEST_JQ), + ['example'], + ), + ], + id='multiple_modules', + ), + pytest.param( + [ + ( + Query('demo.query.*', TEST_JQ), + ['example', 'example2'], + ), + ], + id='multiple_modules_same_collection', + ), + ), +) +def test_save_indirect_host_entries(bare_job, state): + all_task_names = [] + for entry in state: + query, module_names = entry + all_task_names.extend([f'{query.get_fqcn()}.{module_name}' for module_name in module_names]) + query.create_event_queries(module_names) + [query.create_registered_event(bare_job, n) for n in module_names] + save_indirect_host_entries(bare_job.id) + bare_job.refresh_from_db() -@pytest.mark.django_db -def test_save_indirect_host_entries(job_with_counted_event, event_query): - assert job_with_counted_event.event_queries_processed is False - save_indirect_host_entries(job_with_counted_event.id) - job_with_counted_event.refresh_from_db() - assert job_with_counted_event.event_queries_processed is True - assert IndirectManagedNodeAudit.objects.filter(job=job_with_counted_event).count() == 1 - host_audit = IndirectManagedNodeAudit.objects.filter(job=job_with_counted_event).first() - assert host_audit.count == 1 + assert bare_job.event_queries_processed is True + + assert IndirectManagedNodeAudit.objects.filter(job=bare_job).count() == 1 + host_audit = IndirectManagedNodeAudit.objects.filter(job=bare_job).first() + + assert host_audit.count == len(all_task_names) assert host_audit.canonical_facts == {'host_name': 'foo_host'} assert host_audit.facts == {'another_host_name': 'foo_host'} - assert host_audit.organization == job_with_counted_event.organization + assert host_audit.organization == bare_job.organization assert host_audit.name == 'vm-foo' - - -@pytest.mark.django_db -def test_multiple_events_same_module_same_host(bare_job, event_query): - "This tests that the count field gives correct answers" - create_registered_event(bare_job) - create_registered_event(bare_job) - create_registered_event(bare_job) - - save_indirect_host_entries(bare_job.id) - - assert IndirectManagedNodeAudit.objects.filter(job=bare_job).count() == 1 - host_audit = IndirectManagedNodeAudit.objects.filter(job=bare_job).first() - - assert host_audit.count == 3 - assert host_audit.events == ['demo.query.example'] - - -@pytest.mark.django_db -def test_multiple_registered_modules(bare_job): - "This tests that the events will list multiple modules if more than 1 module from different collections is registered and used" - create_registered_event(bare_job, task_name='demo.query.example') - create_registered_event(bare_job, task_name='demo2.query.example') - - # These take the place of using the event_query fixture - create_event_query(fqcn='demo.query') - create_event_query(fqcn='demo2.query') - - save_indirect_host_entries(bare_job.id) - - assert IndirectManagedNodeAudit.objects.filter(job=bare_job).count() == 1 - host_audit = IndirectManagedNodeAudit.objects.filter(job=bare_job).first() - - assert host_audit.count == 2 - assert set(host_audit.events) == {'demo.query.example', 'demo2.query.example'} - - -@pytest.mark.django_db -def test_multiple_registered_modules_same_collection(bare_job): - "This tests that the events will list multiple modules if more than 1 module in same collection is registered and used" - create_registered_event(bare_job, task_name='demo.query.example') - create_registered_event(bare_job, task_name='demo.query.example2') - - # Takes place of event_query fixture, doing manually here - EventQuery.objects.create( - fqcn='demo.query', - collection_version='1.0.1', - event_query=yaml.dump( - { - 'demo.query.example': {'query': TEST_JQ}, - 'demo.query.example2': {'query': TEST_JQ}, - }, - default_flow_style=False, - ), - ) - - save_indirect_host_entries(bare_job.id) - - assert IndirectManagedNodeAudit.objects.filter(job=bare_job).count() == 1 - host_audit = IndirectManagedNodeAudit.objects.filter(job=bare_job).first() - - assert host_audit.count == 2 - assert set(host_audit.events) == {'demo.query.example', 'demo.query.example2'} + assert set(host_audit.events) == set(all_task_names) @pytest.mark.django_db