Host Metrics update/soft delete

awx/api/views/__init__.py

@@ -1562,6 +1562,11 @@ class HostMetricDetail(RetrieveDestroyAPIView):
     serializer_class = serializers.HostMetricSerializer
     permission_classes = (IsSystemAdminOrAuditor,)
 
+    def delete(self, request, *args, **kwargs):
+        self.get_object().soft_delete()
+
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
 
 class HostList(HostRelatedSearchMixin, ListCreateAPIView):
     always_allow_superuser = False

awx/main/models/events.py

@@ -6,7 +6,7 @@ from collections import defaultdict
 
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
-from django.db import models, DatabaseError
+from django.db import connection, models, DatabaseError
 from django.utils.dateparse import parse_datetime
 from django.utils.text import Truncator
 from django.utils.timezone import utc, now
@@ -536,7 +536,7 @@ class JobEvent(BasePlaybookEvent):
                 return
             job = self.job
 
-            from awx.main.models import Host, JobHostSummary, HostMetric  # circular import
+            from awx.main.models import Host, JobHostSummary  # circular import
 
             all_hosts = Host.objects.filter(pk__in=self.host_map.values()).only('id', 'name')
             existing_host_ids = set(h.id for h in all_hosts)
@@ -575,12 +575,26 @@ class JobEvent(BasePlaybookEvent):
 
             Host.objects.bulk_update(list(updated_hosts), ['last_job_id', 'last_job_host_summary_id'], batch_size=100)
 
+            # Create/update Host Metrics
+            self._update_host_metrics(updated_hosts_list)
+
+    @staticmethod
+    def _update_host_metrics(updated_hosts_list):
+        from awx.main.models import HostMetric  # circular import
+
         # bulk-create
         current_time = now()
         HostMetric.objects.bulk_create(
-                [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=100
+            [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=1000
         )
-            HostMetric.objects.filter(hostname__in=updated_hosts_list).update(last_automation=current_time)
+        # bulk-update
+        batch_start, batch_size = 0, 1000
+        while batch_start <= len(updated_hosts_list):
+            batched_host_list = updated_hosts_list[batch_start : (batch_start + batch_size)]
+            HostMetric.objects.filter(hostname__in=batched_host_list).update(
+                last_automation=current_time, automated_counter=models.F('automated_counter') + 1, deleted=False
+            )
+            batch_start += batch_size
 
     @property
     def job_verbosity(self):

awx/main/models/inventory.py

@@ -834,6 +834,18 @@ class HostMetric(models.Model):
     def get_absolute_url(self, request=None):
         return reverse('api:host_metric_detail', kwargs={'pk': self.pk}, request=request)
 
+    def soft_delete(self):
+        if not self.deleted:
+            self.deleted_counter = (self.deleted_counter or 0) + 1
+            self.last_deleted = now()
+            self.deleted = True
+            self.save()
+
+    def soft_restore(self):
+        if self.deleted:
+            self.deleted = False
+            self.save()
+
 
 class InventorySourceOptions(BaseModel):
     """

awx/main/tests/functional/models/test_events.py

@@ -3,12 +3,21 @@ import pytest
 
 from django.utils.timezone import now
 
-from awx.main.models import Job, JobEvent, Inventory, Host, JobHostSummary
+from django.db.models import Q
+
+from awx.main.models import Job, JobEvent, Inventory, Host, JobHostSummary, HostMetric
 
 
 @pytest.mark.django_db
-@mock.patch('awx.main.models.events.emit_event_detail')
+class TestEvents:
-def test_parent_changed(emit):
+    def setup_method(self):
+        self.hostnames = []
+        self.host_map = dict()
+        self.inventory = None
+        self.job = None
+
+    @mock.patch('awx.main.models.events.emit_event_detail')
+    def test_parent_changed(self, emit):
         j = Job()
         j.save()
         JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save()
@@ -24,11 +33,9 @@ def test_parent_changed(emit):
         for e in events.all():
             assert e.changed is True
 
-
+    @pytest.mark.parametrize('event', JobEvent.FAILED_EVENTS)
-@pytest.mark.django_db
+    @mock.patch('awx.main.models.events.emit_event_detail')
-@pytest.mark.parametrize('event', JobEvent.FAILED_EVENTS)
+    def test_parent_failed(self, emit, event):
-@mock.patch('awx.main.models.events.emit_event_detail')
-def test_parent_failed(emit, event):
         j = Job()
         j.save()
         JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save()
@@ -45,38 +52,15 @@ def test_parent_failed(emit, event):
         for e in events.all():
             assert e.failed is True
 
+    def test_host_summary_generation(self):
+        self._generate_hosts(100)
+        self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames))
 
-@pytest.mark.django_db
+        assert self.job.job_host_summaries.count() == len(self.hostnames)
-def test_host_summary_generation():
+        assert sorted([s.host_name for s in self.job.job_host_summaries.all()]) == sorted(self.hostnames)
-    hostnames = [f'Host {i}' for i in range(100)]
-    inv = Inventory()
-    inv.save()
-    Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames])
-    j = Job(inventory=inv)
-    j.save()
-    host_map = dict((host.name, host.id) for host in inv.hosts.all())
-    JobEvent.create_from_data(
-        job_id=j.pk,
-        parent_uuid='abc123',
-        event='playbook_on_stats',
-        event_data={
-            'ok': dict((hostname, len(hostname)) for hostname in hostnames),
-            'changed': {},
-            'dark': {},
-            'failures': {},
-            'ignored': {},
-            'processed': {},
-            'rescued': {},
-            'skipped': {},
-        },
-        host_map=host_map,
-    ).save()
 
-    assert j.job_host_summaries.count() == len(hostnames)
+        for s in self.job.job_host_summaries.all():
-    assert sorted([s.host_name for s in j.job_host_summaries.all()]) == sorted(hostnames)
+            assert self.host_map[s.host_name] == s.host_id
-
-    for s in j.job_host_summaries.all():
-        assert host_map[s.host_name] == s.host_id
             assert s.ok == len(s.host_name)
             assert s.changed == 0
             assert s.dark == 0
@@ -87,84 +71,34 @@ def test_host_summary_generation():
             assert s.skipped == 0
 
         for host in Host.objects.all():
-        assert host.last_job_id == j.id
+            assert host.last_job_id == self.job.id
             assert host.last_job_host_summary.host == host
 
-
+    def test_host_summary_generation_with_deleted_hosts(self):
-@pytest.mark.django_db
+        self._generate_hosts(10)
-def test_host_summary_generation_with_deleted_hosts():
-    hostnames = [f'Host {i}' for i in range(10)]
-    inv = Inventory()
-    inv.save()
-    Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames])
-    j = Job(inventory=inv)
-    j.save()
-    host_map = dict((host.name, host.id) for host in inv.hosts.all())
 
         # delete half of the hosts during the playbook run
-    for h in inv.hosts.all()[:5]:
+        for h in self.inventory.hosts.all()[:5]:
             h.delete()
 
-    JobEvent.create_from_data(
+        self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames))
-        job_id=j.pk,
-        parent_uuid='abc123',
-        event='playbook_on_stats',
-        event_data={
-            'ok': dict((hostname, len(hostname)) for hostname in hostnames),
-            'changed': {},
-            'dark': {},
-            'failures': {},
-            'ignored': {},
-            'processed': {},
-            'rescued': {},
-            'skipped': {},
-        },
-        host_map=host_map,
-    ).save()
 
-    ids = sorted([s.host_id or -1 for s in j.job_host_summaries.order_by('id').all()])
+        ids = sorted([s.host_id or -1 for s in self.job.job_host_summaries.order_by('id').all()])
-    names = sorted([s.host_name for s in j.job_host_summaries.all()])
+        names = sorted([s.host_name for s in self.job.job_host_summaries.all()])
         assert ids == [-1, -1, -1, -1, -1, 6, 7, 8, 9, 10]
         assert names == ['Host 0', 'Host 1', 'Host 2', 'Host 3', 'Host 4', 'Host 5', 'Host 6', 'Host 7', 'Host 8', 'Host 9']
 
-
+    def test_host_summary_generation_with_limit(self):
-@pytest.mark.django_db
-def test_host_summary_generation_with_limit():
         # Make an inventory with 10 hosts, run a playbook with a --limit
         # pointed at *one* host,
         # Verify that *only* that host has an associated JobHostSummary and that
         # *only* that host has an updated value for .last_job.
-    hostnames = [f'Host {i}' for i in range(10)]
+        self._generate_hosts(10)
-    inv = Inventory()
-    inv.save()
-    Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames])
-    j = Job(inventory=inv)
-    j.save()
-
-    # host map is a data structure that tracks a mapping of host name --> ID
-    # for the inventory, _regardless_ of whether or not there's a limit
-    # applied to the actual playbook run
-    host_map = dict((host.name, host.id) for host in inv.hosts.all())
 
         # by making the playbook_on_stats *only* include Host 1, we're emulating
         # the behavior of a `--limit=Host 1`
         matching_host = Host.objects.get(name='Host 1')
-    JobEvent.create_from_data(
+        self._create_job_event(ok={matching_host.name: len(matching_host.name)})  # effectively, limit=Host 1
-        job_id=j.pk,
-        parent_uuid='abc123',
-        event='playbook_on_stats',
-        event_data={
-            'ok': {matching_host.name: len(matching_host.name)},  # effectively, limit=Host 1
-            'changed': {},
-            'dark': {},
-            'failures': {},
-            'ignored': {},
-            'processed': {},
-            'rescued': {},
-            'skipped': {},
-        },
-        host_map=host_map,
-    ).save()
 
         # since the playbook_on_stats only references one host,
         # there should *only* be on JobHostSummary record (and it should
@@ -172,9 +106,106 @@ def test_host_summary_generation_with_limit():
         assert JobHostSummary.objects.count() == 1
         for h in Host.objects.all():
             if h.name == 'Host 1':
-            assert h.last_job_id == j.id
+                assert h.last_job_id == self.job.id
                 assert h.last_job_host_summary_id == JobHostSummary.objects.first().id
             else:
                 # all other hosts in the inventory should remain untouched
                 assert h.last_job_id is None
                 assert h.last_job_host_summary_id is None
+
+    def test_host_metrics_insert(self):
+        self._generate_hosts(10)
+
+        self._create_job_event(
+            ok=dict((hostname, len(hostname)) for hostname in self.hostnames[0:3]),
+            failures=dict((hostname, len(hostname)) for hostname in self.hostnames[3:6]),
+            processed=dict((hostname, len(hostname)) for hostname in self.hostnames[6:9]),
+            skipped=dict((hostname, len(hostname)) for hostname in [self.hostnames[9]]),
+        )
+
+        metrics = HostMetric.objects.all()
+        assert len(metrics) == 10
+        for hm in metrics:
+            assert hm.automated_counter == 1
+            assert hm.last_automation is not None
+            assert hm.deleted is False
+
+    def test_host_metrics_update(self):
+        self._generate_hosts(12)
+
+        self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames))
+
+        # Soft delete 6 host metrics
+        for hm in HostMetric.objects.filter(id__in=[1, 3, 5, 7, 9, 11]):
+            hm.soft_delete()
+
+        assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6
+        assert len(HostMetric.objects.filter(Q(deleted=True) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6
+
+        # hostnames in 'ignored' and 'rescued' stats are ignored
+        self.job = Job(inventory=self.inventory)
+        self.job.save()
+        self._create_job_event(
+            ignored=dict((hostname, len(hostname)) for hostname in self.hostnames[0:6]),
+            rescued=dict((hostname, len(hostname)) for hostname in self.hostnames[6:11]),
+        )
+
+        assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6
+        assert len(HostMetric.objects.filter(Q(deleted=True) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6
+
+        # hostnames in 'changed', 'dark', 'failures', 'ok', 'processed', 'skipped' are processed
+        self.job = Job(inventory=self.inventory)
+        self.job.save()
+        self._create_job_event(
+            changed=dict((hostname, len(hostname)) for hostname in self.hostnames[0:2]),
+            dark=dict((hostname, len(hostname)) for hostname in self.hostnames[2:4]),
+            failures=dict((hostname, len(hostname)) for hostname in self.hostnames[4:6]),
+            ok=dict((hostname, len(hostname)) for hostname in self.hostnames[6:8]),
+            processed=dict((hostname, len(hostname)) for hostname in self.hostnames[8:10]),
+            skipped=dict((hostname, len(hostname)) for hostname in self.hostnames[10:12]),
+        )
+        assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6
+        assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6
+
+    def _generate_hosts(self, cnt, id_from=0):
+        self.hostnames = [f'Host {i}' for i in range(id_from, id_from + cnt)]
+        self.inventory = Inventory()
+        self.inventory.save()
+        Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=self.inventory.id) for h in self.hostnames])
+        self.job = Job(inventory=self.inventory)
+        self.job.save()
+
+        # host map is a data structure that tracks a mapping of host name --> ID
+        # for the inventory, _regardless_ of whether or not there's a limit
+        # applied to the actual playbook run
+        self.host_map = dict((host.name, host.id) for host in self.inventory.hosts.all())
+
+    def _create_job_event(
+        self,
+        parent_uuid='abc123',
+        event='playbook_on_stats',
+        ok=None,
+        changed=None,
+        dark=None,
+        failures=None,
+        ignored=None,
+        processed=None,
+        rescued=None,
+        skipped=None,
+    ):
+        JobEvent.create_from_data(
+            job_id=self.job.pk,
+            parent_uuid=parent_uuid,
+            event=event,
+            event_data={
+                'ok': ok or {},
+                'changed': changed or {},
+                'dark': dark or {},
+                'failures': failures or {},
+                'ignored': ignored or {},
+                'processed': processed or {},
+                'rescued': rescued or {},
+                'skipped': skipped or {},
+            },
+            host_map=self.host_map,
+        ).save()

awx/main/tests/functional/models/test_host_metric.py

@@ -20,3 +20,53 @@ def test_host_metrics_generation():
     date_today = now().strftime('%Y-%m-%d')
     result = HostMetric.objects.filter(first_automation__startswith=date_today).count()
     assert result == len(hostnames)
+
+
+@pytest.mark.django_db
+def test_soft_delete():
+    hostnames = [f'Host to delete {i}' for i in range(2)]
+    current_time = now()
+    HostMetric.objects.bulk_create([HostMetric(hostname=h, last_automation=current_time, automated_counter=42) for h in hostnames])
+
+    hm = HostMetric.objects.get(hostname="Host to delete 0")
+    assert hm.last_deleted is None
+
+    last_deleted = None
+    for _ in range(3):
+        # soft delete 1st
+        # 2nd/3rd delete don't have an effect
+        hm.soft_delete()
+        if last_deleted is None:
+            last_deleted = hm.last_deleted
+
+        assert hm.deleted is True
+        assert hm.deleted_counter == 1
+        assert hm.last_deleted == last_deleted
+        assert hm.automated_counter == 42
+
+    # 2nd record is not touched
+    hm = HostMetric.objects.get(hostname="Host to delete 1")
+    assert hm.deleted is False
+    assert hm.deleted_counter == 0
+    assert hm.last_deleted is None
+    assert hm.automated_counter == 42
+
+
+@pytest.mark.django_db
+def test_soft_restore():
+    current_time = now()
+    HostMetric.objects.create(hostname="Host 1", last_automation=current_time, deleted=True)
+    HostMetric.objects.create(hostname="Host 2", last_automation=current_time, deleted=True, last_deleted=current_time)
+    HostMetric.objects.create(hostname="Host 3", last_automation=current_time, deleted=False, last_deleted=current_time)
+    HostMetric.objects.all().update(automated_counter=42, deleted_counter=10)
+
+    # 1. deleted, last_deleted not null
+    for hm in HostMetric.objects.all():
+        for _ in range(3):
+            hm.soft_restore()
+            assert hm.deleted is False
+            assert hm.automated_counter == 42 and hm.deleted_counter == 10
+            if hm.hostname == "Host 1":
+                assert hm.last_deleted is None
+            else:
+                assert hm.last_deleted == current_time