Aap 41580 indirect host count wildcard query (#15893)

* Support <collection_namespace>.<collection_name>.* indirect host query
  to match ANY module in the <collection_namespace>.<collection_name>
* Add tests for new wildcard indirect host count
* error checking of ansible event name
* error checking of ansible event query

awx/main/tasks/host_indirect.py

@@ -45,22 +45,35 @@ def build_indirect_host_data(job: Job, job_event_queries: dict[str, dict[str, st
     facts_missing_logged = False
     unhashable_facts_logged = False
 
+    job_event_queries_fqcn = {}
+    for query_k, query_v in job_event_queries.items():
+        if len(parts := query_k.split('.')) != 3:
+            logger.info(f"Skiping malformed query '{query_k}'. Expected to be of the form 'a.b.c'")
+            continue
+        if parts[2] != '*':
+            continue
+        job_event_queries_fqcn['.'.join(parts[0:2])] = query_v
+
     for event in job.job_events.filter(event_data__isnull=False).iterator():
         if 'res' not in event.event_data:
             continue
 
-        if 'resolved_action' not in event.event_data or event.event_data['resolved_action'] not in job_event_queries.keys():
+        if not (resolved_action := event.event_data.get('resolved_action', None)):
             continue
 
-        resolved_action = event.event_data['resolved_action']
+        if len(resolved_action_parts := resolved_action.split('.')) != 3:
+            logger.debug(f"Malformed invocation module name '{resolved_action}'. Expected to be of the form 'a.b.c'")
+            continue
 
-        # We expect a dict with a 'query' key for the resolved_action
+        resolved_action_fqcn = '.'.join(resolved_action_parts[0:2])
-        if 'query' not in job_event_queries[resolved_action]:
+
+        # Match module invocation to collection queries
+        # First match against fully qualified query names i.e. a.b.c
+        # Then try and match against wildcard queries i.e. a.b.*
+        if not (jq_str_for_event := job_event_queries.get(resolved_action, job_event_queries_fqcn.get(resolved_action_fqcn, {})).get('query')):
             continue
 
         # Recall from cache, or process the jq expression, and loop over the jq results
-        jq_str_for_event = job_event_queries[resolved_action]['query']
-
         if jq_str_for_event not in compiled_jq_expressions:
             compiled_jq_expressions[resolved_action] = jq.compile(jq_str_for_event)
         compiled_jq = compiled_jq_expressions[resolved_action]

awx/main/tests/functional/tasks/test_host_indirect.py

@@ -1,4 +1,5 @@
 import yaml
+from functools import reduce
 from unittest import mock
 
 import pytest
@@ -20,6 +21,46 @@ from awx.main.models.indirect_managed_node_audit import IndirectManagedNodeAudit
 TEST_JQ = "{name: .name, canonical_facts: {host_name: .direct_host_name}, facts: {another_host_name: .direct_host_name}}"
 
 
+class Query(dict):
+    def __init__(self, resolved_action: str, query_jq: dict):
+        self._resolved_action = resolved_action.split('.')
+        self._collection_ns, self._collection_name, self._module_name = self._resolved_action
+
+        super().__init__({self.resolve_key: {'query': query_jq}})
+
+    def get_fqcn(self):
+        return f'{self._collection_ns}.{self._collection_name}'
+
+    @property
+    def resolve_value(self):
+        return self[self.resolve_key]
+
+    @property
+    def resolve_key(self):
+        return f'{self.get_fqcn()}.{self._module_name}'
+
+    def resolve(self, module_name=None):
+        return {f'{self.get_fqcn()}.{module_name or self._module_name}': self.resolve_value}
+
+    def create_event_query(self, module_name=None):
+        if (module_name := module_name or self._module_name) == '*':
+            raise ValueError('Invalid module name *')
+        return self.create_event_queries([module_name])
+
+    def create_event_queries(self, module_names):
+        queries = {}
+        for name in module_names:
+            queries |= self.resolve(name)
+        return EventQuery.objects.create(
+            fqcn=self.get_fqcn(),
+            collection_version='1.0.1',
+            event_query=yaml.dump(queries, default_flow_style=False),
+        )
+
+    def create_registered_event(self, job, module_name):
+        job.job_events.create(event_data={'resolved_action': f'{self.get_fqcn()}.{module_name}', 'res': {'direct_host_name': 'foo_host', 'name': 'vm-foo'}})
+
+
 @pytest.fixture
 def bare_job(job_factory):
     job = job_factory()
@@ -39,11 +80,6 @@ def job_with_counted_event(bare_job):
     return bare_job
 
 
-def create_event_query(fqcn='demo.query'):
-    module_name = f'{fqcn}.example'
-    return EventQuery.objects.create(fqcn=fqcn, collection_version='1.0.1', event_query=yaml.dump({module_name: {'query': TEST_JQ}}, default_flow_style=False))
-
-
 def create_audit_record(name, job, organization, created=now()):
     record = IndirectManagedNodeAudit.objects.create(name=name, job=job, organization=organization)
     record.created = created
@@ -54,7 +90,7 @@ def create_audit_record(name, job, organization, created=now()):
 @pytest.fixture
 def event_query():
     "This is ordinarily created by the artifacts callback"
-    return create_event_query()
+    return Query('demo.query.example', TEST_JQ).create_event_query()
 
 
 @pytest.fixture
@@ -72,105 +108,211 @@ def new_audit_record(bare_job, organization):
 
 
 @pytest.mark.django_db
-def test_build_with_no_results(bare_job):
+@pytest.mark.parametrize(
-    # never filled in events, should do nothing
+    'queries,expected_matches',
-    assert build_indirect_host_data(bare_job, {}) == []
+    (
+        pytest.param(
+            [],
+            0,
+            id='no_results',
+        ),
+        pytest.param(
+            [Query('demo.query.example', TEST_JQ)],
+            1,
+            id='fully_qualified',
+        ),
+        pytest.param(
+            [Query('demo.query.*', TEST_JQ)],
+            1,
+            id='wildcard',
+        ),
+        pytest.param(
+            [
+                Query('demo.query.*', TEST_JQ),
+                Query('demo.query.example', TEST_JQ),
+            ],
+            1,
+            id='wildcard_and_fully_qualified',
+        ),
+        pytest.param(
+            [
+                Query('demo.query.*', TEST_JQ),
+                Query('demo.query.example', {}),
+            ],
+            0,
+            id='wildcard_and_fully_qualified',
+        ),
+        pytest.param(
+            [
+                Query('demo.query.example', {}),
+                Query('demo.query.*', TEST_JQ),
+            ],
+            0,
+            id='ordering_should_not_matter',
+        ),
+    ),
+)
+def test_build_indirect_host_data(job_with_counted_event, queries: Query, expected_matches: int):
+    data = build_indirect_host_data(job_with_counted_event, {k: v for d in queries for k, v in d.items()})
+    assert len(data) == expected_matches
+
+
+@mock.patch('awx.main.tasks.host_indirect.logger.debug')
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    'task_name',
+    (
+        pytest.param(
+            'demo.query',
+            id='no_results',
+        ),
+        pytest.param(
+            'demo',
+            id='no_results',
+        ),
+        pytest.param(
+            'a.b.c.d',
+            id='no_results',
+        ),
+    ),
+)
+def test_build_indirect_host_data_malformed_module_name(mock_logger_debug, bare_job, task_name: str):
+    create_registered_event(bare_job, task_name)
+    assert build_indirect_host_data(bare_job, Query('demo.query.example', TEST_JQ)) == []
+    mock_logger_debug.assert_called_once_with(f"Malformed invocation module name '{task_name}'. Expected to be of the form 'a.b.c'")
+
+
+@mock.patch('awx.main.tasks.host_indirect.logger.info')
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    'query',
+    (
+        pytest.param(
+            'demo.query',
+            id='no_results',
+        ),
+        pytest.param(
+            'demo',
+            id='no_results',
+        ),
+        pytest.param(
+            'a.b.c.d',
+            id='no_results',
+        ),
+    ),
+)
+def test_build_indirect_host_data_malformed_query(mock_logger_info, job_with_counted_event, query: str):
+    assert build_indirect_host_data(job_with_counted_event, {query: {'query': TEST_JQ}}) == []
+    mock_logger_info.assert_called_once_with(f"Skiping malformed query '{query}'. Expected to be of the form 'a.b.c'")
 
 
 @pytest.mark.django_db
-def test_collect_an_event(job_with_counted_event):
+@pytest.mark.parametrize(
-    records = build_indirect_host_data(job_with_counted_event, {'demo.query.example': {'query': TEST_JQ}})
+    'query',
-    assert len(records) == 1
+    (
+        pytest.param(
+            Query('demo.query.example', TEST_JQ),
+            id='fully_qualified',
+        ),
+        pytest.param(
+            Query('demo.query.*', TEST_JQ),
+            id='wildcard',
+        ),
+    ),
+)
+def test_fetch_job_event_query(bare_job, query: Query):
+    query.create_event_query(module_name='example')
+    assert fetch_job_event_query(bare_job) == query.resolve('example')
 
 
 @pytest.mark.django_db
-def test_fetch_job_event_query(bare_job, event_query):
+@pytest.mark.parametrize(
-    assert fetch_job_event_query(bare_job) == {'demo.query.example': {'query': TEST_JQ}}
+    'queries',
+    (
+        [
+            Query('demo.query.example', TEST_JQ),
+            Query('demo2.query.example', TEST_JQ),
+        ],
+        [
+            Query('demo.query.*', TEST_JQ),
+            Query('demo2.query.example', TEST_JQ),
+        ],
+    ),
+)
+def test_fetch_multiple_job_event_query(bare_job, queries: list[Query]):
+    for q in queries:
+        q.create_event_query(module_name='example')
+    assert fetch_job_event_query(bare_job) == reduce(lambda acc, q: acc | q.resolve('example'), queries, {})
 
 
 @pytest.mark.django_db
-def test_fetch_multiple_job_event_query(bare_job):
+@pytest.mark.parametrize(
-    create_event_query(fqcn='demo.query')
+    ('state',),
-    create_event_query(fqcn='demo2.query')
+    (
-    assert fetch_job_event_query(bare_job) == {'demo.query.example': {'query': TEST_JQ}, 'demo2.query.example': {'query': TEST_JQ}}
+        pytest.param(
+            [
+                (
+                    Query('demo.query.example', TEST_JQ),
+                    ['example'],
+                ),
+            ],
+            id='fully_qualified',
+        ),
+        pytest.param(
+            [
+                (
+                    Query('demo.query.example', TEST_JQ),
+                    ['example'] * 3,
+                ),
+            ],
+            id='multiple_events_same_module_same_host',
+        ),
+        pytest.param(
+            [
+                (
+                    Query('demo.query.example', TEST_JQ),
+                    ['example'],
+                ),
+                (
+                    Query('demo2.query.example', TEST_JQ),
+                    ['example'],
+                ),
+            ],
+            id='multiple_modules',
+        ),
+        pytest.param(
+            [
+                (
+                    Query('demo.query.*', TEST_JQ),
+                    ['example', 'example2'],
+                ),
+            ],
+            id='multiple_modules_same_collection',
+        ),
+    ),
+)
+def test_save_indirect_host_entries(bare_job, state):
+    all_task_names = []
+    for entry in state:
+        query, module_names = entry
+        all_task_names.extend([f'{query.get_fqcn()}.{module_name}' for module_name in module_names])
+        query.create_event_queries(module_names)
+        [query.create_registered_event(bare_job, n) for n in module_names]
 
+    save_indirect_host_entries(bare_job.id)
+    bare_job.refresh_from_db()
 
-@pytest.mark.django_db
+    assert bare_job.event_queries_processed is True
-def test_save_indirect_host_entries(job_with_counted_event, event_query):
+
-    assert job_with_counted_event.event_queries_processed is False
+    assert IndirectManagedNodeAudit.objects.filter(job=bare_job).count() == 1
-    save_indirect_host_entries(job_with_counted_event.id)
+    host_audit = IndirectManagedNodeAudit.objects.filter(job=bare_job).first()
-    job_with_counted_event.refresh_from_db()
+
-    assert job_with_counted_event.event_queries_processed is True
+    assert host_audit.count == len(all_task_names)
-    assert IndirectManagedNodeAudit.objects.filter(job=job_with_counted_event).count() == 1
-    host_audit = IndirectManagedNodeAudit.objects.filter(job=job_with_counted_event).first()
-    assert host_audit.count == 1
     assert host_audit.canonical_facts == {'host_name': 'foo_host'}
     assert host_audit.facts == {'another_host_name': 'foo_host'}
-    assert host_audit.organization == job_with_counted_event.organization
+    assert host_audit.organization == bare_job.organization
     assert host_audit.name == 'vm-foo'
-
+    assert set(host_audit.events) == set(all_task_names)
-
-@pytest.mark.django_db
-def test_multiple_events_same_module_same_host(bare_job, event_query):
-    "This tests that the count field gives correct answers"
-    create_registered_event(bare_job)
-    create_registered_event(bare_job)
-    create_registered_event(bare_job)
-
-    save_indirect_host_entries(bare_job.id)
-
-    assert IndirectManagedNodeAudit.objects.filter(job=bare_job).count() == 1
-    host_audit = IndirectManagedNodeAudit.objects.filter(job=bare_job).first()
-
-    assert host_audit.count == 3
-    assert host_audit.events == ['demo.query.example']
-
-
-@pytest.mark.django_db
-def test_multiple_registered_modules(bare_job):
-    "This tests that the events will list multiple modules if more than 1 module from different collections is registered and used"
-    create_registered_event(bare_job, task_name='demo.query.example')
-    create_registered_event(bare_job, task_name='demo2.query.example')
-
-    # These take the place of using the event_query fixture
-    create_event_query(fqcn='demo.query')
-    create_event_query(fqcn='demo2.query')
-
-    save_indirect_host_entries(bare_job.id)
-
-    assert IndirectManagedNodeAudit.objects.filter(job=bare_job).count() == 1
-    host_audit = IndirectManagedNodeAudit.objects.filter(job=bare_job).first()
-
-    assert host_audit.count == 2
-    assert set(host_audit.events) == {'demo.query.example', 'demo2.query.example'}
-
-
-@pytest.mark.django_db
-def test_multiple_registered_modules_same_collection(bare_job):
-    "This tests that the events will list multiple modules if more than 1 module in same collection is registered and used"
-    create_registered_event(bare_job, task_name='demo.query.example')
-    create_registered_event(bare_job, task_name='demo.query.example2')
-
-    # Takes place of event_query fixture, doing manually here
-    EventQuery.objects.create(
-        fqcn='demo.query',
-        collection_version='1.0.1',
-        event_query=yaml.dump(
-            {
-                'demo.query.example': {'query': TEST_JQ},
-                'demo.query.example2': {'query': TEST_JQ},
-            },
-            default_flow_style=False,
-        ),
-    )
-
-    save_indirect_host_entries(bare_job.id)
-
-    assert IndirectManagedNodeAudit.objects.filter(job=bare_job).count() == 1
-    host_audit = IndirectManagedNodeAudit.objects.filter(job=bare_job).first()
-
-    assert host_audit.count == 2
-    assert set(host_audit.events) == {'demo.query.example', 'demo.query.example2'}
 
 
 @pytest.mark.django_db