From ef4e77d78f5c1bb45040c0460c105fa4fde247a2 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Thu, 2 Feb 2023 15:17:14 +0100 Subject: [PATCH 01/28] Host Metrics List API --- awx/api/serializers.py | 9 +++++++++ awx/api/urls/host_metric.py | 10 ++++++++++ awx/api/urls/urls.py | 2 ++ awx/api/views/__init__.py | 7 +++++++ awx/main/access.py | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 61 insertions(+) create mode 100644 awx/api/urls/host_metric.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index c2f5dfca23..7770838735 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -56,6 +56,7 @@ from awx.main.models import ( ExecutionEnvironment, Group, Host, + HostMetric, Instance, InstanceGroup, InstanceLink, @@ -5386,6 +5387,14 @@ class InstanceHealthCheckSerializer(BaseSerializer): fields = read_only_fields +class HostMetricSerializer(BaseSerializer): + show_capabilities = ['delete'] + + class Meta: + model = HostMetric + fields = ("hostname", "first_automation", "last_automation") + + class InstanceGroupSerializer(BaseSerializer): show_capabilities = ['edit', 'delete'] capacity = serializers.SerializerMethodField() diff --git a/awx/api/urls/host_metric.py b/awx/api/urls/host_metric.py new file mode 100644 index 0000000000..547c995c10 --- /dev/null +++ b/awx/api/urls/host_metric.py @@ -0,0 +1,10 @@ +# Copyright (c) 2017 Ansible, Inc. +# All Rights Reserved. + +from django.urls import re_path + +from awx.api.views import HostMetricList + +urls = [re_path(r'$^', HostMetricList.as_view(), name='host_metric_list')] + +__all__ = ['urls'] diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index 15d9fdd2ca..b9dbafca33 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -50,6 +50,7 @@ from .inventory import urls as inventory_urls from .execution_environments import urls as execution_environment_urls from .team import urls as team_urls from .host import urls as host_urls +from .host_metric import urls as host_metric_urls from .group import urls as group_urls from .inventory_source import urls as inventory_source_urls from .inventory_update import urls as inventory_update_urls @@ -118,6 +119,7 @@ v2_urls = [ re_path(r'^teams/', include(team_urls)), re_path(r'^inventories/', include(inventory_urls)), re_path(r'^hosts/', include(host_urls)), + re_path(r'^host_metrics/', include(host_metric_urls)), re_path(r'^groups/', include(group_urls)), re_path(r'^inventory_sources/', include(inventory_source_urls)), re_path(r'^inventory_updates/', include(inventory_update_urls)), diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index fc3fe52610..729f993b74 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -1548,6 +1548,13 @@ class HostRelatedSearchMixin(object): return ret +class HostMetricList(ListAPIView): + always_allow_superuser = False + name = _("Host Metrics List") + model = models.HostMetric + serializer_class = serializers.HostMetricSerializer + + class HostList(HostRelatedSearchMixin, ListCreateAPIView): always_allow_superuser = False model = models.Host diff --git a/awx/main/access.py b/awx/main/access.py index 5d51ab3b91..938619119e 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -37,6 +37,7 @@ from awx.main.models import ( ExecutionEnvironment, Group, Host, + HostMetric, Instance, InstanceGroup, Inventory, @@ -883,6 +884,38 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess): return super(OrganizationAccess, self).can_attach(obj, sub_obj, relationship, *args, **kwargs) +class HostMetricAccess(BaseAccess): + """ + - I can see host metrics when a super user or system auditor. + - I can delete host metrics when a super user. + """ + + model = HostMetric + + def get_queryset(self): + # if self.user.is_superuser or self.user.is_system_auditor: + # return self.model.objects.filter(Q(user__isnull=True) | Q(user=self.user)) + # else: + # return self.model.objects.filter(user=self.user) + if self.user.is_superuser or self.user.is_system_auditor: + qs = self.model.objects.all() + else: + qs = self.filtered_queryset() + return qs + + def can_read(self, obj): + return bool(self.user.is_superuser or self.user.is_system_auditor or (obj and obj.user == self.user)) + + def can_add(self, data): + return False # There is no API endpoint to POST new settings. + + def can_change(self, obj, data): + return False + + def can_delete(self, obj): + return bool(self.user.is_superuser or (obj and obj.user == self.user)) + + class InventoryAccess(BaseAccess): """ I can see inventory when: From d80759cd7ac3d042bbe3ff7e37132ee56181cc98 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Fri, 3 Feb 2023 14:22:21 +0100 Subject: [PATCH 02/28] HostMetrics migration --- awx/api/serializers.py | 13 +++++- awx/api/urls/host_metric.py | 4 +- awx/api/views/__init__.py | 10 ++++- awx/main/access.py | 8 +--- .../migrations/0175_add_hostmetric_fields.py | 43 +++++++++++++++++++ awx/main/models/inventory.py | 12 +++++- 6 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 awx/main/migrations/0175_add_hostmetric_fields.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 7770838735..5989adb4db 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -5392,7 +5392,18 @@ class HostMetricSerializer(BaseSerializer): class Meta: model = HostMetric - fields = ("hostname", "first_automation", "last_automation") + fields = ( + "id", + "hostname", + "url", + "first_automation", + "last_automation", + "last_deleted", + "automated_counter", + "deleted_counter", + "deleted", + "used_in_inventories", + ) class InstanceGroupSerializer(BaseSerializer): diff --git a/awx/api/urls/host_metric.py b/awx/api/urls/host_metric.py index 547c995c10..d464fb82c5 100644 --- a/awx/api/urls/host_metric.py +++ b/awx/api/urls/host_metric.py @@ -3,8 +3,8 @@ from django.urls import re_path -from awx.api.views import HostMetricList +from awx.api.views import HostMetricList, HostMetricDetail -urls = [re_path(r'$^', HostMetricList.as_view(), name='host_metric_list')] +urls = [re_path(r'$^', HostMetricList.as_view(), name='host_metric_list'), re_path(r'^(?P[0-9]+)/$', HostMetricDetail.as_view(), name='host_metric_detail')] __all__ = ['urls'] diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 729f993b74..1845b55c2e 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -1549,10 +1549,18 @@ class HostRelatedSearchMixin(object): class HostMetricList(ListAPIView): - always_allow_superuser = False name = _("Host Metrics List") model = models.HostMetric serializer_class = serializers.HostMetricSerializer + permission_classes = (IsSystemAdminOrAuditor,) + search_fields = ('hostname', 'deleted') + + +class HostMetricDetail(RetrieveDestroyAPIView): + name = _("Host Metric Detail") + model = models.HostMetric + serializer_class = serializers.HostMetricSerializer + permission_classes = (IsSystemAdminOrAuditor,) class HostList(HostRelatedSearchMixin, ListCreateAPIView): diff --git a/awx/main/access.py b/awx/main/access.py index 938619119e..185ab1e061 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -886,17 +886,13 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess): class HostMetricAccess(BaseAccess): """ - - I can see host metrics when a super user or system auditor. - - I can delete host metrics when a super user. + - I can see host metrics when I'm a super user or system auditor. + - I can delete host metrics when I'm a super user. """ model = HostMetric def get_queryset(self): - # if self.user.is_superuser or self.user.is_system_auditor: - # return self.model.objects.filter(Q(user__isnull=True) | Q(user=self.user)) - # else: - # return self.model.objects.filter(user=self.user) if self.user.is_superuser or self.user.is_system_auditor: qs = self.model.objects.all() else: diff --git a/awx/main/migrations/0175_add_hostmetric_fields.py b/awx/main/migrations/0175_add_hostmetric_fields.py new file mode 100644 index 0000000000..d273a6b6ea --- /dev/null +++ b/awx/main/migrations/0175_add_hostmetric_fields.py @@ -0,0 +1,43 @@ +# Generated by Django 3.2.16 on 2023-02-03 09:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('main', '0174_ensure_org_ee_admin_roles'), + ] + + operations = [ + migrations.AlterField(model_name='hostmetric', name='hostname', field=models.CharField(max_length=512, primary_key=False, serialize=True, unique=True)), + migrations.AddField( + model_name='hostmetric', + name='last_deleted', + field=models.DateTimeField(db_index=True, null=True, help_text='When the host was last deleted'), + ), + migrations.AddField( + model_name='hostmetric', + name='automated_counter', + field=models.BigIntegerField(default=0, help_text='How many times was the host automated'), + ), + migrations.AddField( + model_name='hostmetric', + name='deleted_counter', + field=models.BigIntegerField(default=0, help_text='How many times was the host deleted'), + ), + migrations.AddField( + model_name='hostmetric', + name='deleted', + field=models.BooleanField( + default=False, help_text='Boolean flag saying whether the host is deleted and therefore not counted into the subscription consumption' + ), + ), + migrations.AddField( + model_name='hostmetric', + name='used_in_inventories', + field=models.BigIntegerField(null=True, help_text='How many inventories contain this host'), + ), + migrations.AddField( + model_name='hostmetric', name='id', field=models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID') + ), + ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 7b55c51851..577c03645c 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -820,9 +820,19 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin): class HostMetric(models.Model): - hostname = models.CharField(primary_key=True, max_length=512) + hostname = models.CharField(unique=True, max_length=512) first_automation = models.DateTimeField(auto_now_add=True, null=False, db_index=True, help_text=_('When the host was first automated against')) last_automation = models.DateTimeField(db_index=True, help_text=_('When the host was last automated against')) + last_deleted = models.DateTimeField(null=True, db_index=True, help_text=_('When the host was last deleted')) + automated_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host automated')) + deleted_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host deleted')) + deleted = models.BooleanField( + default=False, help_text=_('Boolean flag saying whether the host is deleted and therefore not counted into the subscription consumption') + ) + used_in_inventories = models.BigIntegerField(null=True, help_text=_('How many inventories contain this host')) + + def get_absolute_url(self, request=None): + return reverse('api:host_metric_detail', kwargs={'pk': self.pk}, request=request) class InventorySourceOptions(BaseModel): From b18ad77035a0ee58f7517a940c7ce4ceca4ddf50 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Thu, 9 Feb 2023 16:39:39 +0100 Subject: [PATCH 03/28] Host Metrics update/soft delete --- awx/api/views/__init__.py | 5 + awx/main/models/events.py | 28 +- awx/main/models/inventory.py | 12 + .../tests/functional/models/test_events.py | 357 ++++++++++-------- .../functional/models/test_host_metric.py | 50 +++ 5 files changed, 282 insertions(+), 170 deletions(-) diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 1845b55c2e..7dda3d4d44 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -1562,6 +1562,11 @@ class HostMetricDetail(RetrieveDestroyAPIView): serializer_class = serializers.HostMetricSerializer permission_classes = (IsSystemAdminOrAuditor,) + def delete(self, request, *args, **kwargs): + self.get_object().soft_delete() + + return Response(status=status.HTTP_204_NO_CONTENT) + class HostList(HostRelatedSearchMixin, ListCreateAPIView): always_allow_superuser = False diff --git a/awx/main/models/events.py b/awx/main/models/events.py index 1827802812..0ba07185f4 100644 --- a/awx/main/models/events.py +++ b/awx/main/models/events.py @@ -6,7 +6,7 @@ from collections import defaultdict from django.conf import settings from django.core.exceptions import ObjectDoesNotExist -from django.db import models, DatabaseError +from django.db import connection, models, DatabaseError from django.utils.dateparse import parse_datetime from django.utils.text import Truncator from django.utils.timezone import utc, now @@ -536,7 +536,7 @@ class JobEvent(BasePlaybookEvent): return job = self.job - from awx.main.models import Host, JobHostSummary, HostMetric # circular import + from awx.main.models import Host, JobHostSummary # circular import all_hosts = Host.objects.filter(pk__in=self.host_map.values()).only('id', 'name') existing_host_ids = set(h.id for h in all_hosts) @@ -575,12 +575,26 @@ class JobEvent(BasePlaybookEvent): Host.objects.bulk_update(list(updated_hosts), ['last_job_id', 'last_job_host_summary_id'], batch_size=100) - # bulk-create - current_time = now() - HostMetric.objects.bulk_create( - [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=100 + # Create/update Host Metrics + self._update_host_metrics(updated_hosts_list) + + @staticmethod + def _update_host_metrics(updated_hosts_list): + from awx.main.models import HostMetric # circular import + + # bulk-create + current_time = now() + HostMetric.objects.bulk_create( + [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=1000 + ) + # bulk-update + batch_start, batch_size = 0, 1000 + while batch_start <= len(updated_hosts_list): + batched_host_list = updated_hosts_list[batch_start : (batch_start + batch_size)] + HostMetric.objects.filter(hostname__in=batched_host_list).update( + last_automation=current_time, automated_counter=models.F('automated_counter') + 1, deleted=False ) - HostMetric.objects.filter(hostname__in=updated_hosts_list).update(last_automation=current_time) + batch_start += batch_size @property def job_verbosity(self): diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 577c03645c..4ae7ba2adb 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -834,6 +834,18 @@ class HostMetric(models.Model): def get_absolute_url(self, request=None): return reverse('api:host_metric_detail', kwargs={'pk': self.pk}, request=request) + def soft_delete(self): + if not self.deleted: + self.deleted_counter = (self.deleted_counter or 0) + 1 + self.last_deleted = now() + self.deleted = True + self.save() + + def soft_restore(self): + if self.deleted: + self.deleted = False + self.save() + class InventorySourceOptions(BaseModel): """ diff --git a/awx/main/tests/functional/models/test_events.py b/awx/main/tests/functional/models/test_events.py index 758e69b641..48adc781e7 100644 --- a/awx/main/tests/functional/models/test_events.py +++ b/awx/main/tests/functional/models/test_events.py @@ -3,178 +3,209 @@ import pytest from django.utils.timezone import now -from awx.main.models import Job, JobEvent, Inventory, Host, JobHostSummary +from django.db.models import Q + +from awx.main.models import Job, JobEvent, Inventory, Host, JobHostSummary, HostMetric @pytest.mark.django_db -@mock.patch('awx.main.models.events.emit_event_detail') -def test_parent_changed(emit): - j = Job() - j.save() - JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save() - assert JobEvent.objects.count() == 1 - for e in JobEvent.objects.all(): - assert e.changed is False +class TestEvents: + def setup_method(self): + self.hostnames = [] + self.host_map = dict() + self.inventory = None + self.job = None - JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='runner_on_ok', event_data={'res': {'changed': ['localhost']}}).save() - # the `playbook_on_stats` event is where we update the parent changed linkage - JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save() - events = JobEvent.objects.filter(event__in=['playbook_on_task_start', 'runner_on_ok']) - assert events.count() == 2 - for e in events.all(): - assert e.changed is True + @mock.patch('awx.main.models.events.emit_event_detail') + def test_parent_changed(self, emit): + j = Job() + j.save() + JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save() + assert JobEvent.objects.count() == 1 + for e in JobEvent.objects.all(): + assert e.changed is False + JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='runner_on_ok', event_data={'res': {'changed': ['localhost']}}).save() + # the `playbook_on_stats` event is where we update the parent changed linkage + JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save() + events = JobEvent.objects.filter(event__in=['playbook_on_task_start', 'runner_on_ok']) + assert events.count() == 2 + for e in events.all(): + assert e.changed is True -@pytest.mark.django_db -@pytest.mark.parametrize('event', JobEvent.FAILED_EVENTS) -@mock.patch('awx.main.models.events.emit_event_detail') -def test_parent_failed(emit, event): - j = Job() - j.save() - JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save() - assert JobEvent.objects.count() == 1 - for e in JobEvent.objects.all(): - assert e.failed is False + @pytest.mark.parametrize('event', JobEvent.FAILED_EVENTS) + @mock.patch('awx.main.models.events.emit_event_detail') + def test_parent_failed(self, emit, event): + j = Job() + j.save() + JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save() + assert JobEvent.objects.count() == 1 + for e in JobEvent.objects.all(): + assert e.failed is False - JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event=event).save() + JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event=event).save() - # the `playbook_on_stats` event is where we update the parent failed linkage - JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save() - events = JobEvent.objects.filter(event__in=['playbook_on_task_start', event]) - assert events.count() == 2 - for e in events.all(): - assert e.failed is True + # the `playbook_on_stats` event is where we update the parent failed linkage + JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save() + events = JobEvent.objects.filter(event__in=['playbook_on_task_start', event]) + assert events.count() == 2 + for e in events.all(): + assert e.failed is True + def test_host_summary_generation(self): + self._generate_hosts(100) + self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames)) -@pytest.mark.django_db -def test_host_summary_generation(): - hostnames = [f'Host {i}' for i in range(100)] - inv = Inventory() - inv.save() - Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames]) - j = Job(inventory=inv) - j.save() - host_map = dict((host.name, host.id) for host in inv.hosts.all()) - JobEvent.create_from_data( - job_id=j.pk, + assert self.job.job_host_summaries.count() == len(self.hostnames) + assert sorted([s.host_name for s in self.job.job_host_summaries.all()]) == sorted(self.hostnames) + + for s in self.job.job_host_summaries.all(): + assert self.host_map[s.host_name] == s.host_id + assert s.ok == len(s.host_name) + assert s.changed == 0 + assert s.dark == 0 + assert s.failures == 0 + assert s.ignored == 0 + assert s.processed == 0 + assert s.rescued == 0 + assert s.skipped == 0 + + for host in Host.objects.all(): + assert host.last_job_id == self.job.id + assert host.last_job_host_summary.host == host + + def test_host_summary_generation_with_deleted_hosts(self): + self._generate_hosts(10) + + # delete half of the hosts during the playbook run + for h in self.inventory.hosts.all()[:5]: + h.delete() + + self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames)) + + ids = sorted([s.host_id or -1 for s in self.job.job_host_summaries.order_by('id').all()]) + names = sorted([s.host_name for s in self.job.job_host_summaries.all()]) + assert ids == [-1, -1, -1, -1, -1, 6, 7, 8, 9, 10] + assert names == ['Host 0', 'Host 1', 'Host 2', 'Host 3', 'Host 4', 'Host 5', 'Host 6', 'Host 7', 'Host 8', 'Host 9'] + + def test_host_summary_generation_with_limit(self): + # Make an inventory with 10 hosts, run a playbook with a --limit + # pointed at *one* host, + # Verify that *only* that host has an associated JobHostSummary and that + # *only* that host has an updated value for .last_job. + self._generate_hosts(10) + + # by making the playbook_on_stats *only* include Host 1, we're emulating + # the behavior of a `--limit=Host 1` + matching_host = Host.objects.get(name='Host 1') + self._create_job_event(ok={matching_host.name: len(matching_host.name)}) # effectively, limit=Host 1 + + # since the playbook_on_stats only references one host, + # there should *only* be on JobHostSummary record (and it should + # be related to the appropriate Host) + assert JobHostSummary.objects.count() == 1 + for h in Host.objects.all(): + if h.name == 'Host 1': + assert h.last_job_id == self.job.id + assert h.last_job_host_summary_id == JobHostSummary.objects.first().id + else: + # all other hosts in the inventory should remain untouched + assert h.last_job_id is None + assert h.last_job_host_summary_id is None + + def test_host_metrics_insert(self): + self._generate_hosts(10) + + self._create_job_event( + ok=dict((hostname, len(hostname)) for hostname in self.hostnames[0:3]), + failures=dict((hostname, len(hostname)) for hostname in self.hostnames[3:6]), + processed=dict((hostname, len(hostname)) for hostname in self.hostnames[6:9]), + skipped=dict((hostname, len(hostname)) for hostname in [self.hostnames[9]]), + ) + + metrics = HostMetric.objects.all() + assert len(metrics) == 10 + for hm in metrics: + assert hm.automated_counter == 1 + assert hm.last_automation is not None + assert hm.deleted is False + + def test_host_metrics_update(self): + self._generate_hosts(12) + + self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames)) + + # Soft delete 6 host metrics + for hm in HostMetric.objects.filter(id__in=[1, 3, 5, 7, 9, 11]): + hm.soft_delete() + + assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6 + assert len(HostMetric.objects.filter(Q(deleted=True) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6 + + # hostnames in 'ignored' and 'rescued' stats are ignored + self.job = Job(inventory=self.inventory) + self.job.save() + self._create_job_event( + ignored=dict((hostname, len(hostname)) for hostname in self.hostnames[0:6]), + rescued=dict((hostname, len(hostname)) for hostname in self.hostnames[6:11]), + ) + + assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6 + assert len(HostMetric.objects.filter(Q(deleted=True) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6 + + # hostnames in 'changed', 'dark', 'failures', 'ok', 'processed', 'skipped' are processed + self.job = Job(inventory=self.inventory) + self.job.save() + self._create_job_event( + changed=dict((hostname, len(hostname)) for hostname in self.hostnames[0:2]), + dark=dict((hostname, len(hostname)) for hostname in self.hostnames[2:4]), + failures=dict((hostname, len(hostname)) for hostname in self.hostnames[4:6]), + ok=dict((hostname, len(hostname)) for hostname in self.hostnames[6:8]), + processed=dict((hostname, len(hostname)) for hostname in self.hostnames[8:10]), + skipped=dict((hostname, len(hostname)) for hostname in self.hostnames[10:12]), + ) + assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6 + assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6 + + def _generate_hosts(self, cnt, id_from=0): + self.hostnames = [f'Host {i}' for i in range(id_from, id_from + cnt)] + self.inventory = Inventory() + self.inventory.save() + Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=self.inventory.id) for h in self.hostnames]) + self.job = Job(inventory=self.inventory) + self.job.save() + + # host map is a data structure that tracks a mapping of host name --> ID + # for the inventory, _regardless_ of whether or not there's a limit + # applied to the actual playbook run + self.host_map = dict((host.name, host.id) for host in self.inventory.hosts.all()) + + def _create_job_event( + self, parent_uuid='abc123', event='playbook_on_stats', - event_data={ - 'ok': dict((hostname, len(hostname)) for hostname in hostnames), - 'changed': {}, - 'dark': {}, - 'failures': {}, - 'ignored': {}, - 'processed': {}, - 'rescued': {}, - 'skipped': {}, - }, - host_map=host_map, - ).save() - - assert j.job_host_summaries.count() == len(hostnames) - assert sorted([s.host_name for s in j.job_host_summaries.all()]) == sorted(hostnames) - - for s in j.job_host_summaries.all(): - assert host_map[s.host_name] == s.host_id - assert s.ok == len(s.host_name) - assert s.changed == 0 - assert s.dark == 0 - assert s.failures == 0 - assert s.ignored == 0 - assert s.processed == 0 - assert s.rescued == 0 - assert s.skipped == 0 - - for host in Host.objects.all(): - assert host.last_job_id == j.id - assert host.last_job_host_summary.host == host - - -@pytest.mark.django_db -def test_host_summary_generation_with_deleted_hosts(): - hostnames = [f'Host {i}' for i in range(10)] - inv = Inventory() - inv.save() - Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames]) - j = Job(inventory=inv) - j.save() - host_map = dict((host.name, host.id) for host in inv.hosts.all()) - - # delete half of the hosts during the playbook run - for h in inv.hosts.all()[:5]: - h.delete() - - JobEvent.create_from_data( - job_id=j.pk, - parent_uuid='abc123', - event='playbook_on_stats', - event_data={ - 'ok': dict((hostname, len(hostname)) for hostname in hostnames), - 'changed': {}, - 'dark': {}, - 'failures': {}, - 'ignored': {}, - 'processed': {}, - 'rescued': {}, - 'skipped': {}, - }, - host_map=host_map, - ).save() - - ids = sorted([s.host_id or -1 for s in j.job_host_summaries.order_by('id').all()]) - names = sorted([s.host_name for s in j.job_host_summaries.all()]) - assert ids == [-1, -1, -1, -1, -1, 6, 7, 8, 9, 10] - assert names == ['Host 0', 'Host 1', 'Host 2', 'Host 3', 'Host 4', 'Host 5', 'Host 6', 'Host 7', 'Host 8', 'Host 9'] - - -@pytest.mark.django_db -def test_host_summary_generation_with_limit(): - # Make an inventory with 10 hosts, run a playbook with a --limit - # pointed at *one* host, - # Verify that *only* that host has an associated JobHostSummary and that - # *only* that host has an updated value for .last_job. - hostnames = [f'Host {i}' for i in range(10)] - inv = Inventory() - inv.save() - Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames]) - j = Job(inventory=inv) - j.save() - - # host map is a data structure that tracks a mapping of host name --> ID - # for the inventory, _regardless_ of whether or not there's a limit - # applied to the actual playbook run - host_map = dict((host.name, host.id) for host in inv.hosts.all()) - - # by making the playbook_on_stats *only* include Host 1, we're emulating - # the behavior of a `--limit=Host 1` - matching_host = Host.objects.get(name='Host 1') - JobEvent.create_from_data( - job_id=j.pk, - parent_uuid='abc123', - event='playbook_on_stats', - event_data={ - 'ok': {matching_host.name: len(matching_host.name)}, # effectively, limit=Host 1 - 'changed': {}, - 'dark': {}, - 'failures': {}, - 'ignored': {}, - 'processed': {}, - 'rescued': {}, - 'skipped': {}, - }, - host_map=host_map, - ).save() - - # since the playbook_on_stats only references one host, - # there should *only* be on JobHostSummary record (and it should - # be related to the appropriate Host) - assert JobHostSummary.objects.count() == 1 - for h in Host.objects.all(): - if h.name == 'Host 1': - assert h.last_job_id == j.id - assert h.last_job_host_summary_id == JobHostSummary.objects.first().id - else: - # all other hosts in the inventory should remain untouched - assert h.last_job_id is None - assert h.last_job_host_summary_id is None + ok=None, + changed=None, + dark=None, + failures=None, + ignored=None, + processed=None, + rescued=None, + skipped=None, + ): + JobEvent.create_from_data( + job_id=self.job.pk, + parent_uuid=parent_uuid, + event=event, + event_data={ + 'ok': ok or {}, + 'changed': changed or {}, + 'dark': dark or {}, + 'failures': failures or {}, + 'ignored': ignored or {}, + 'processed': processed or {}, + 'rescued': rescued or {}, + 'skipped': skipped or {}, + }, + host_map=self.host_map, + ).save() diff --git a/awx/main/tests/functional/models/test_host_metric.py b/awx/main/tests/functional/models/test_host_metric.py index 1f560e474f..dad8295435 100644 --- a/awx/main/tests/functional/models/test_host_metric.py +++ b/awx/main/tests/functional/models/test_host_metric.py @@ -20,3 +20,53 @@ def test_host_metrics_generation(): date_today = now().strftime('%Y-%m-%d') result = HostMetric.objects.filter(first_automation__startswith=date_today).count() assert result == len(hostnames) + + +@pytest.mark.django_db +def test_soft_delete(): + hostnames = [f'Host to delete {i}' for i in range(2)] + current_time = now() + HostMetric.objects.bulk_create([HostMetric(hostname=h, last_automation=current_time, automated_counter=42) for h in hostnames]) + + hm = HostMetric.objects.get(hostname="Host to delete 0") + assert hm.last_deleted is None + + last_deleted = None + for _ in range(3): + # soft delete 1st + # 2nd/3rd delete don't have an effect + hm.soft_delete() + if last_deleted is None: + last_deleted = hm.last_deleted + + assert hm.deleted is True + assert hm.deleted_counter == 1 + assert hm.last_deleted == last_deleted + assert hm.automated_counter == 42 + + # 2nd record is not touched + hm = HostMetric.objects.get(hostname="Host to delete 1") + assert hm.deleted is False + assert hm.deleted_counter == 0 + assert hm.last_deleted is None + assert hm.automated_counter == 42 + + +@pytest.mark.django_db +def test_soft_restore(): + current_time = now() + HostMetric.objects.create(hostname="Host 1", last_automation=current_time, deleted=True) + HostMetric.objects.create(hostname="Host 2", last_automation=current_time, deleted=True, last_deleted=current_time) + HostMetric.objects.create(hostname="Host 3", last_automation=current_time, deleted=False, last_deleted=current_time) + HostMetric.objects.all().update(automated_counter=42, deleted_counter=10) + + # 1. deleted, last_deleted not null + for hm in HostMetric.objects.all(): + for _ in range(3): + hm.soft_restore() + assert hm.deleted is False + assert hm.automated_counter == 42 and hm.deleted_counter == 10 + if hm.hostname == "Host 1": + assert hm.last_deleted is None + else: + assert hm.last_deleted == current_time From 05f918e666e86b84c2d89d0a47fe60dee1b123a0 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Fri, 10 Feb 2023 10:29:28 +0100 Subject: [PATCH 04/28] HostMetric compliance computation --- awx/main/managers.py | 5 +++++ awx/main/models/inventory.py | 5 ++++- awx/main/utils/licensing.py | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/awx/main/managers.py b/awx/main/managers.py index 32d6ed7f5b..b674049b0e 100644 --- a/awx/main/managers.py +++ b/awx/main/managers.py @@ -79,6 +79,11 @@ class HostManager(models.Manager): return qs +class HostMetricActiveManager(models.Manager): + def get_queryset(self): + return super().get_queryset().filter(deleted=False) + + def get_ig_ig_mapping(ig_instance_mapping, instance_ig_mapping): # Create IG mapping by union of all groups their instances are members of ig_ig_mapping = {} diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 4ae7ba2adb..41a85b0071 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -32,7 +32,7 @@ from awx.main.fields import ( SmartFilterField, OrderedManyToManyField, ) -from awx.main.managers import HostManager +from awx.main.managers import HostManager, HostMetricActiveManager from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, prevent_search, accepts_json from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate @@ -831,6 +831,9 @@ class HostMetric(models.Model): ) used_in_inventories = models.BigIntegerField(null=True, help_text=_('How many inventories contain this host')) + objects = models.Manager() + active_objects = HostMetricActiveManager() + def get_absolute_url(self, request=None): return reverse('api:host_metric_detail', kwargs={'pk': self.pk}, request=request) diff --git a/awx/main/utils/licensing.py b/awx/main/utils/licensing.py index bec953f822..3bc5e174e5 100644 --- a/awx/main/utils/licensing.py +++ b/awx/main/utils/licensing.py @@ -382,8 +382,8 @@ class Licenser(object): current_instances = Host.objects.active_count() license_date = int(attrs.get('license_date', 0) or 0) - automated_instances = HostMetric.objects.count() - first_host = HostMetric.objects.only('first_automation').order_by('first_automation').first() + automated_instances = HostMetric.active_objects.count() + first_host = HostMetric.active_objects.only('first_automation').order_by('first_automation').first() if first_host: automated_since = int(first_host.first_automation.timestamp()) else: From f919178734c420c6af00862ae03a653720ccfc38 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Mon, 13 Feb 2023 13:54:11 +0100 Subject: [PATCH 05/28] HostMetricSummaryMonthly API and Migrations --- awx/api/filters.py | 4 +++ awx/api/serializers.py | 8 +++++ awx/api/urls/urls.py | 2 ++ awx/api/views/__init__.py | 34 ++++++++++++++++++- awx/main/access.py | 28 +++++++++++++++ .../0176_hostmetricsummarymonthly.py | 33 ++++++++++++++++++ awx/main/models/__init__.py | 1 + awx/main/models/inventory.py | 15 +++++++- 8 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 awx/main/migrations/0176_hostmetricsummarymonthly.py diff --git a/awx/api/filters.py b/awx/api/filters.py index 1a6e3eb90e..e574fce539 100644 --- a/awx/api/filters.py +++ b/awx/api/filters.py @@ -385,6 +385,10 @@ class FieldLookupBackend(BaseFilterBackend): raise ParseError(json.dumps(e.messages, ensure_ascii=False)) +class HostMetricSummaryMonthlyFieldLookupBackend(FieldLookupBackend): + RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by', 'search', 'type', 'past_months', 'count_disabled', 'no_truncate', 'limit') + + class OrderByBackend(BaseFilterBackend): """ Filter to apply ordering based on query string parameters. diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 5989adb4db..1ff00854c2 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -57,6 +57,7 @@ from awx.main.models import ( Group, Host, HostMetric, + HostMetricSummaryMonthly, Instance, InstanceGroup, InstanceLink, @@ -5406,6 +5407,13 @@ class HostMetricSerializer(BaseSerializer): ) +class HostMetricSummaryMonthlySerializer(BaseSerializer): + class Meta: + model = HostMetricSummaryMonthly + read_only_fields = ("id", "date", "license_consumed", "license_capacity", "hosts_added", "hosts_deleted", "indirectly_managed_hosts") + fields = read_only_fields + + class InstanceGroupSerializer(BaseSerializer): show_capabilities = ['edit', 'delete'] capacity = serializers.SerializerMethodField() diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index b9dbafca33..c7d73165c3 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -30,6 +30,7 @@ from awx.api.views import ( OAuth2TokenList, ApplicationOAuth2TokenList, OAuth2ApplicationDetail, + HostMetricSummaryMonthlyList, ) from awx.api.views.bulk import ( @@ -120,6 +121,7 @@ v2_urls = [ re_path(r'^inventories/', include(inventory_urls)), re_path(r'^hosts/', include(host_urls)), re_path(r'^host_metrics/', include(host_metric_urls)), + re_path(r'^host_metric_summary_monthly/$', HostMetricSummaryMonthlyList.as_view(), name='host_metric_summary_monthly_list'), re_path(r'^groups/', include(group_urls)), re_path(r'^inventory_sources/', include(inventory_source_urls)), re_path(r'^inventory_updates/', include(inventory_update_urls)), diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 7dda3d4d44..0dd0f306cd 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -3,6 +3,7 @@ # Python import dateutil +import datetime import functools import html import itertools @@ -17,7 +18,6 @@ from collections import OrderedDict from urllib3.exceptions import ConnectTimeoutError - # Django from django.conf import settings from django.core.exceptions import FieldError, ObjectDoesNotExist @@ -122,6 +122,7 @@ from awx.api.views.mixin import ( UnifiedJobDeletionMixin, NoTruncateMixin, ) +from awx.api.filters import HostMetricSummaryMonthlyFieldLookupBackend from awx.api.pagination import UnifiedJobEventPagination from awx.main.utils import set_environ @@ -1568,6 +1569,37 @@ class HostMetricDetail(RetrieveDestroyAPIView): return Response(status=status.HTTP_204_NO_CONTENT) +class HostMetricSummaryMonthlyList(ListAPIView): + name = _("Host Metrics Summary Monthly") + model = models.HostMetricSummaryMonthly + permission_classes = (IsSystemAdminOrAuditor,) + serializer_class = serializers.HostMetricSummaryMonthlySerializer + search_fields = ('date',) + filter_backends = [HostMetricSummaryMonthlyFieldLookupBackend] + + def get_queryset(self): + queryset = super().get_queryset() + past_months = self.request.query_params.get('past_months', None) + date_from = self._get_date_from(past_months) + + queryset = queryset.filter(date__gte=date_from) + return queryset + + @staticmethod + def _get_date_from(past_months, default=12, maximum=36): + try: + months_ago = int(past_months or default) + except ValueError: + months_ago = default + months_ago = min(months_ago, maximum) + months_ago = max(months_ago, 1) + + date_from = datetime.date.today() - dateutil.relativedelta.relativedelta(months=months_ago) + # Set to beginning of the month + date_from = date_from.replace(day=1).isoformat() + return date_from + + class HostList(HostRelatedSearchMixin, ListCreateAPIView): always_allow_superuser = False model = models.Host diff --git a/awx/main/access.py b/awx/main/access.py index 185ab1e061..5cca1ff133 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -38,6 +38,7 @@ from awx.main.models import ( Group, Host, HostMetric, + HostMetricSummaryMonthly, Instance, InstanceGroup, Inventory, @@ -912,6 +913,33 @@ class HostMetricAccess(BaseAccess): return bool(self.user.is_superuser or (obj and obj.user == self.user)) +class HostMetricSummaryMonthlyAccess(BaseAccess): + """ + - I can see host metrics when I'm a super user or system auditor. + """ + + model = HostMetricSummaryMonthly + + def get_queryset(self): + if self.user.is_superuser or self.user.is_system_auditor: + qs = self.model.objects.all() + else: + qs = self.filtered_queryset() + return qs + + def can_read(self, obj): + return bool(self.user.is_superuser or self.user.is_system_auditor or (obj and obj.user == self.user)) + + def can_add(self, data): + return False # There is no API endpoint to POST new settings. + + def can_change(self, obj, data): + return False + + def can_delete(self, obj): + return False + + class InventoryAccess(BaseAccess): """ I can see inventory when: diff --git a/awx/main/migrations/0176_hostmetricsummarymonthly.py b/awx/main/migrations/0176_hostmetricsummarymonthly.py new file mode 100644 index 0000000000..735f46f0d6 --- /dev/null +++ b/awx/main/migrations/0176_hostmetricsummarymonthly.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.16 on 2023-02-10 12:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('main', '0175_add_hostmetric_fields'), + ] + + operations = [ + migrations.CreateModel( + name='HostMetricSummaryMonthly', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date', models.DateField(unique=True)), + ('license_consumed', models.BigIntegerField(default=0, help_text='How much unique hosts are consumed from the license')), + ('license_capacity', models.BigIntegerField(default=0, help_text="'License capacity as max. number of unique hosts")), + ( + 'hosts_added', + models.BigIntegerField(default=0, help_text='How many hosts were added in the associated month, consuming more license capacity'), + ), + ( + 'hosts_deleted', + models.BigIntegerField(default=0, help_text='How many hosts were deleted in the associated month, freeing the license capacity'), + ), + ( + 'indirectly_managed_hosts', + models.BigIntegerField(default=0, help_text='Manually entered number indirectly managed hosts for a certain month'), + ), + ], + ), + ] diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index ed49b98083..8a608aeead 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -16,6 +16,7 @@ from awx.main.models.inventory import ( # noqa Group, Host, HostMetric, + HostMetricSummaryMonthly, Inventory, InventorySource, InventoryUpdate, diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 41a85b0071..c937c4342d 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -53,7 +53,7 @@ from awx.main.utils.execution_environments import to_container_path from awx.main.utils.licensing import server_product_name -__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'SmartInventoryMembership'] +__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'SmartInventoryMembership', 'HostMetric', 'HostMetricSummaryMonthly'] logger = logging.getLogger('awx.main.models.inventory') @@ -850,6 +850,19 @@ class HostMetric(models.Model): self.save() +class HostMetricSummaryMonthly(models.Model): + """ + HostMetric summaries computed by scheduled task monthly + """ + + date = models.DateField(unique=True) + license_consumed = models.BigIntegerField(default=0, help_text=_("How much unique hosts are consumed from the license")) + license_capacity = models.BigIntegerField(default=0, help_text=_("'License capacity as max. number of unique hosts")) + hosts_added = models.BigIntegerField(default=0, help_text=_("How many hosts were added in the associated month, consuming more license capacity")) + hosts_deleted = models.BigIntegerField(default=0, help_text=_("How many hosts were deleted in the associated month, freeing the license capacity")) + indirectly_managed_hosts = models.BigIntegerField(default=0, help_text=("Manually entered number indirectly managed hosts for a certain month")) + + class InventorySourceOptions(BaseModel): """ Common fields for InventorySource and InventoryUpdate. From e6050804f98e7ab566f29f80970c701f96062063 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Wed, 15 Feb 2023 16:49:43 +0100 Subject: [PATCH 06/28] HostMetric review,migration,permissions --- awx/api/urls/host_metric.py | 2 +- awx/api/views/__init__.py | 7 ++- awx/api/views/root.py | 2 + awx/main/access.py | 57 ------------------- .../migrations/0175_add_hostmetric_fields.py | 6 +- .../0176_hostmetricsummarymonthly.py | 6 +- awx/main/models/events.py | 4 +- awx/main/models/inventory.py | 16 +++--- 8 files changed, 24 insertions(+), 76 deletions(-) diff --git a/awx/api/urls/host_metric.py b/awx/api/urls/host_metric.py index d464fb82c5..a5e43fefbc 100644 --- a/awx/api/urls/host_metric.py +++ b/awx/api/urls/host_metric.py @@ -5,6 +5,6 @@ from django.urls import re_path from awx.api.views import HostMetricList, HostMetricDetail -urls = [re_path(r'$^', HostMetricList.as_view(), name='host_metric_list'), re_path(r'^(?P[0-9]+)/$', HostMetricDetail.as_view(), name='host_metric_detail')] +urls = [re_path(r'^$', HostMetricList.as_view(), name='host_metric_list'), re_path(r'^(?P[0-9]+)/$', HostMetricDetail.as_view(), name='host_metric_detail')] __all__ = ['urls'] diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 0dd0f306cd..d05806afb1 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -1556,6 +1556,9 @@ class HostMetricList(ListAPIView): permission_classes = (IsSystemAdminOrAuditor,) search_fields = ('hostname', 'deleted') + def get_queryset(self): + return self.model.objects.all() + class HostMetricDetail(RetrieveDestroyAPIView): name = _("Host Metric Detail") @@ -1572,13 +1575,13 @@ class HostMetricDetail(RetrieveDestroyAPIView): class HostMetricSummaryMonthlyList(ListAPIView): name = _("Host Metrics Summary Monthly") model = models.HostMetricSummaryMonthly - permission_classes = (IsSystemAdminOrAuditor,) serializer_class = serializers.HostMetricSummaryMonthlySerializer + permission_classes = (IsSystemAdminOrAuditor,) search_fields = ('date',) filter_backends = [HostMetricSummaryMonthlyFieldLookupBackend] def get_queryset(self): - queryset = super().get_queryset() + queryset = self.model.objects.all() past_months = self.request.query_params.get('past_months', None) date_from = self._get_date_from(past_months) diff --git a/awx/api/views/root.py b/awx/api/views/root.py index 7b4eb8191e..f343d8169d 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -102,6 +102,8 @@ class ApiVersionRootView(APIView): data['inventory_updates'] = reverse('api:inventory_update_list', request=request) data['groups'] = reverse('api:group_list', request=request) data['hosts'] = reverse('api:host_list', request=request) + data['host_metrics'] = reverse('api:host_metric_list', request=request) + data['host_metric_summary_monthly'] = reverse('api:host_metric_summary_monthly_list', request=request) data['job_templates'] = reverse('api:job_template_list', request=request) data['jobs'] = reverse('api:job_list', request=request) data['ad_hoc_commands'] = reverse('api:ad_hoc_command_list', request=request) diff --git a/awx/main/access.py b/awx/main/access.py index 5cca1ff133..5d51ab3b91 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -37,8 +37,6 @@ from awx.main.models import ( ExecutionEnvironment, Group, Host, - HostMetric, - HostMetricSummaryMonthly, Instance, InstanceGroup, Inventory, @@ -885,61 +883,6 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess): return super(OrganizationAccess, self).can_attach(obj, sub_obj, relationship, *args, **kwargs) -class HostMetricAccess(BaseAccess): - """ - - I can see host metrics when I'm a super user or system auditor. - - I can delete host metrics when I'm a super user. - """ - - model = HostMetric - - def get_queryset(self): - if self.user.is_superuser or self.user.is_system_auditor: - qs = self.model.objects.all() - else: - qs = self.filtered_queryset() - return qs - - def can_read(self, obj): - return bool(self.user.is_superuser or self.user.is_system_auditor or (obj and obj.user == self.user)) - - def can_add(self, data): - return False # There is no API endpoint to POST new settings. - - def can_change(self, obj, data): - return False - - def can_delete(self, obj): - return bool(self.user.is_superuser or (obj and obj.user == self.user)) - - -class HostMetricSummaryMonthlyAccess(BaseAccess): - """ - - I can see host metrics when I'm a super user or system auditor. - """ - - model = HostMetricSummaryMonthly - - def get_queryset(self): - if self.user.is_superuser or self.user.is_system_auditor: - qs = self.model.objects.all() - else: - qs = self.filtered_queryset() - return qs - - def can_read(self, obj): - return bool(self.user.is_superuser or self.user.is_system_auditor or (obj and obj.user == self.user)) - - def can_add(self, data): - return False # There is no API endpoint to POST new settings. - - def can_change(self, obj, data): - return False - - def can_delete(self, obj): - return False - - class InventoryAccess(BaseAccess): """ I can see inventory when: diff --git a/awx/main/migrations/0175_add_hostmetric_fields.py b/awx/main/migrations/0175_add_hostmetric_fields.py index d273a6b6ea..ee91b01fbb 100644 --- a/awx/main/migrations/0175_add_hostmetric_fields.py +++ b/awx/main/migrations/0175_add_hostmetric_fields.py @@ -18,12 +18,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='hostmetric', name='automated_counter', - field=models.BigIntegerField(default=0, help_text='How many times was the host automated'), + field=models.IntegerField(default=0, help_text='How many times was the host automated'), ), migrations.AddField( model_name='hostmetric', name='deleted_counter', - field=models.BigIntegerField(default=0, help_text='How many times was the host deleted'), + field=models.IntegerField(default=0, help_text='How many times was the host deleted'), ), migrations.AddField( model_name='hostmetric', @@ -35,7 +35,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='hostmetric', name='used_in_inventories', - field=models.BigIntegerField(null=True, help_text='How many inventories contain this host'), + field=models.IntegerField(null=True, help_text='How many inventories contain this host'), ), migrations.AddField( model_name='hostmetric', name='id', field=models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID') diff --git a/awx/main/migrations/0176_hostmetricsummarymonthly.py b/awx/main/migrations/0176_hostmetricsummarymonthly.py index 735f46f0d6..7631363a4c 100644 --- a/awx/main/migrations/0176_hostmetricsummarymonthly.py +++ b/awx/main/migrations/0176_hostmetricsummarymonthly.py @@ -18,15 +18,15 @@ class Migration(migrations.Migration): ('license_capacity', models.BigIntegerField(default=0, help_text="'License capacity as max. number of unique hosts")), ( 'hosts_added', - models.BigIntegerField(default=0, help_text='How many hosts were added in the associated month, consuming more license capacity'), + models.IntegerField(default=0, help_text='How many hosts were added in the associated month, consuming more license capacity'), ), ( 'hosts_deleted', - models.BigIntegerField(default=0, help_text='How many hosts were deleted in the associated month, freeing the license capacity'), + models.IntegerField(default=0, help_text='How many hosts were deleted in the associated month, freeing the license capacity'), ), ( 'indirectly_managed_hosts', - models.BigIntegerField(default=0, help_text='Manually entered number indirectly managed hosts for a certain month'), + models.IntegerField(default=0, help_text='Manually entered number indirectly managed hosts for a certain month'), ), ], ), diff --git a/awx/main/models/events.py b/awx/main/models/events.py index 0ba07185f4..2d6dee6f61 100644 --- a/awx/main/models/events.py +++ b/awx/main/models/events.py @@ -6,7 +6,7 @@ from collections import defaultdict from django.conf import settings from django.core.exceptions import ObjectDoesNotExist -from django.db import connection, models, DatabaseError +from django.db import models, DatabaseError from django.utils.dateparse import parse_datetime from django.utils.text import Truncator from django.utils.timezone import utc, now @@ -585,7 +585,7 @@ class JobEvent(BasePlaybookEvent): # bulk-create current_time = now() HostMetric.objects.bulk_create( - [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=1000 + [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=100 ) # bulk-update batch_start, batch_size = 0, 1000 diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index c937c4342d..56dea84d96 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -824,12 +824,12 @@ class HostMetric(models.Model): first_automation = models.DateTimeField(auto_now_add=True, null=False, db_index=True, help_text=_('When the host was first automated against')) last_automation = models.DateTimeField(db_index=True, help_text=_('When the host was last automated against')) last_deleted = models.DateTimeField(null=True, db_index=True, help_text=_('When the host was last deleted')) - automated_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host automated')) - deleted_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host deleted')) + automated_counter = models.IntegerField(default=0, help_text=_('How many times was the host automated')) + deleted_counter = models.IntegerField(default=0, help_text=_('How many times was the host deleted')) deleted = models.BooleanField( default=False, help_text=_('Boolean flag saying whether the host is deleted and therefore not counted into the subscription consumption') ) - used_in_inventories = models.BigIntegerField(null=True, help_text=_('How many inventories contain this host')) + used_in_inventories = models.IntegerField(null=True, help_text=_('How many inventories contain this host')) objects = models.Manager() active_objects = HostMetricActiveManager() @@ -842,12 +842,12 @@ class HostMetric(models.Model): self.deleted_counter = (self.deleted_counter or 0) + 1 self.last_deleted = now() self.deleted = True - self.save() + self.save(update_fields=['deleted', 'deleted_counter', 'last_deleted']) def soft_restore(self): if self.deleted: self.deleted = False - self.save() + self.save(update_fields=['deleted']) class HostMetricSummaryMonthly(models.Model): @@ -858,9 +858,9 @@ class HostMetricSummaryMonthly(models.Model): date = models.DateField(unique=True) license_consumed = models.BigIntegerField(default=0, help_text=_("How much unique hosts are consumed from the license")) license_capacity = models.BigIntegerField(default=0, help_text=_("'License capacity as max. number of unique hosts")) - hosts_added = models.BigIntegerField(default=0, help_text=_("How many hosts were added in the associated month, consuming more license capacity")) - hosts_deleted = models.BigIntegerField(default=0, help_text=_("How many hosts were deleted in the associated month, freeing the license capacity")) - indirectly_managed_hosts = models.BigIntegerField(default=0, help_text=("Manually entered number indirectly managed hosts for a certain month")) + hosts_added = models.IntegerField(default=0, help_text=_("How many hosts were added in the associated month, consuming more license capacity")) + hosts_deleted = models.IntegerField(default=0, help_text=_("How many hosts were deleted in the associated month, freeing the license capacity")) + indirectly_managed_hosts = models.IntegerField(default=0, help_text=("Manually entered number indirectly managed hosts for a certain month")) class InventorySourceOptions(BaseModel): From e38f87eb1d830a12a9f56ee549aa4f0244a9e7b1 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 23 Feb 2023 14:41:53 -0500 Subject: [PATCH 07/28] Remove custom API filters and suggest solution via templates --- awx/api/filters.py | 4 ---- awx/api/templates/api/host_metric_detail.md | 18 +++++++++++++++ .../api/host_metric_summary_monthly_list.md | 12 ++++++++++ awx/api/views/__init__.py | 23 +------------------ 4 files changed, 31 insertions(+), 26 deletions(-) create mode 100644 awx/api/templates/api/host_metric_detail.md create mode 100644 awx/api/templates/api/host_metric_summary_monthly_list.md diff --git a/awx/api/filters.py b/awx/api/filters.py index e574fce539..1a6e3eb90e 100644 --- a/awx/api/filters.py +++ b/awx/api/filters.py @@ -385,10 +385,6 @@ class FieldLookupBackend(BaseFilterBackend): raise ParseError(json.dumps(e.messages, ensure_ascii=False)) -class HostMetricSummaryMonthlyFieldLookupBackend(FieldLookupBackend): - RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by', 'search', 'type', 'past_months', 'count_disabled', 'no_truncate', 'limit') - - class OrderByBackend(BaseFilterBackend): """ Filter to apply ordering based on query string parameters. diff --git a/awx/api/templates/api/host_metric_detail.md b/awx/api/templates/api/host_metric_detail.md new file mode 100644 index 0000000000..0a59a1b410 --- /dev/null +++ b/awx/api/templates/api/host_metric_detail.md @@ -0,0 +1,18 @@ +{% ifmeth GET %} +# Retrieve {{ model_verbose_name|title|anora }}: + +Make GET request to this resource to retrieve a single {{ model_verbose_name }} +record containing the following fields: + +{% include "api/_result_fields_common.md" %} +{% endifmeth %} + +{% ifmeth DELETE %} +# Delete {{ model_verbose_name|title|anora }}: + +Make a DELETE request to this resource to soft-delete this {{ model_verbose_name }}. + +A soft deletion will mark the `deleted` field as true and exclude the host +metric from license calculations. +This may be undone later if the same hostname is automated again afterwards. +{% endifmeth %} diff --git a/awx/api/templates/api/host_metric_summary_monthly_list.md b/awx/api/templates/api/host_metric_summary_monthly_list.md new file mode 100644 index 0000000000..953b1827a6 --- /dev/null +++ b/awx/api/templates/api/host_metric_summary_monthly_list.md @@ -0,0 +1,12 @@ +# Intended Use Case + +To get summaries from a certain day or earlier, you can filter this +endpoint in the following way. + + ?date__gte=2023-01-01 + +This will return summaries that were produced on that date or later. +These host metric monthly summaries should be automatically produced +by a background task that runs once each month. + +{% include "api/list_api_view.md" %} diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index d05806afb1..0068eb0148 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -122,7 +122,6 @@ from awx.api.views.mixin import ( UnifiedJobDeletionMixin, NoTruncateMixin, ) -from awx.api.filters import HostMetricSummaryMonthlyFieldLookupBackend from awx.api.pagination import UnifiedJobEventPagination from awx.main.utils import set_environ @@ -1578,29 +1577,9 @@ class HostMetricSummaryMonthlyList(ListAPIView): serializer_class = serializers.HostMetricSummaryMonthlySerializer permission_classes = (IsSystemAdminOrAuditor,) search_fields = ('date',) - filter_backends = [HostMetricSummaryMonthlyFieldLookupBackend] def get_queryset(self): - queryset = self.model.objects.all() - past_months = self.request.query_params.get('past_months', None) - date_from = self._get_date_from(past_months) - - queryset = queryset.filter(date__gte=date_from) - return queryset - - @staticmethod - def _get_date_from(past_months, default=12, maximum=36): - try: - months_ago = int(past_months or default) - except ValueError: - months_ago = default - months_ago = min(months_ago, maximum) - months_ago = max(months_ago, 1) - - date_from = datetime.date.today() - dateutil.relativedelta.relativedelta(months=months_ago) - # Set to beginning of the month - date_from = date_from.replace(day=1).isoformat() - return date_from + return self.model.objects.all() class HostList(HostRelatedSearchMixin, ListCreateAPIView): From 7285d82f0000e2a25f5ae065c64b13e0e2b7c49e Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Mon, 27 Feb 2023 11:25:09 +0100 Subject: [PATCH 08/28] HostMetric migration --- awx/api/views/__init__.py | 1 - awx/main/migrations/0175_add_hostmetric_fields.py | 2 +- awx/main/migrations/0176_hostmetricsummarymonthly.py | 2 +- awx/main/models/inventory.py | 4 ++-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 0068eb0148..a9f2e0ce25 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -3,7 +3,6 @@ # Python import dateutil -import datetime import functools import html import itertools diff --git a/awx/main/migrations/0175_add_hostmetric_fields.py b/awx/main/migrations/0175_add_hostmetric_fields.py index ee91b01fbb..75090bd678 100644 --- a/awx/main/migrations/0175_add_hostmetric_fields.py +++ b/awx/main/migrations/0175_add_hostmetric_fields.py @@ -18,7 +18,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='hostmetric', name='automated_counter', - field=models.IntegerField(default=0, help_text='How many times was the host automated'), + field=models.BigIntegerField(default=0, help_text='How many times was the host automated'), ), migrations.AddField( model_name='hostmetric', diff --git a/awx/main/migrations/0176_hostmetricsummarymonthly.py b/awx/main/migrations/0176_hostmetricsummarymonthly.py index 7631363a4c..fe482aa416 100644 --- a/awx/main/migrations/0176_hostmetricsummarymonthly.py +++ b/awx/main/migrations/0176_hostmetricsummarymonthly.py @@ -14,7 +14,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('date', models.DateField(unique=True)), - ('license_consumed', models.BigIntegerField(default=0, help_text='How much unique hosts are consumed from the license')), + ('license_consumed', models.BigIntegerField(default=0, help_text='How many unique hosts are consumed from the license')), ('license_capacity', models.BigIntegerField(default=0, help_text="'License capacity as max. number of unique hosts")), ( 'hosts_added', diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 56dea84d96..829017ee1d 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -824,7 +824,7 @@ class HostMetric(models.Model): first_automation = models.DateTimeField(auto_now_add=True, null=False, db_index=True, help_text=_('When the host was first automated against')) last_automation = models.DateTimeField(db_index=True, help_text=_('When the host was last automated against')) last_deleted = models.DateTimeField(null=True, db_index=True, help_text=_('When the host was last deleted')) - automated_counter = models.IntegerField(default=0, help_text=_('How many times was the host automated')) + automated_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host automated')) deleted_counter = models.IntegerField(default=0, help_text=_('How many times was the host deleted')) deleted = models.BooleanField( default=False, help_text=_('Boolean flag saying whether the host is deleted and therefore not counted into the subscription consumption') @@ -856,7 +856,7 @@ class HostMetricSummaryMonthly(models.Model): """ date = models.DateField(unique=True) - license_consumed = models.BigIntegerField(default=0, help_text=_("How much unique hosts are consumed from the license")) + license_consumed = models.BigIntegerField(default=0, help_text=_("How many unique hosts are consumed from the license")) license_capacity = models.BigIntegerField(default=0, help_text=_("'License capacity as max. number of unique hosts")) hosts_added = models.IntegerField(default=0, help_text=_("How many hosts were added in the associated month, consuming more license capacity")) hosts_deleted = models.IntegerField(default=0, help_text=_("How many hosts were deleted in the associated month, freeing the license capacity")) From 9badbf0b4e45a31f5ce943785c67d7437d914b82 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Mon, 27 Feb 2023 15:20:47 +0100 Subject: [PATCH 09/28] Compliance computation settings --- awx/main/conf.py | 13 ++++++++++++- awx/main/constants.py | 4 ++++ awx/main/utils/licensing.py | 12 ++++++++++-- awx/settings/defaults.py | 6 ++++++ 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/awx/main/conf.py b/awx/main/conf.py index 6634271b93..04dd056f45 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -10,7 +10,7 @@ from rest_framework import serializers # AWX from awx.conf import fields, register, register_validate from awx.main.models import ExecutionEnvironment - +from awx.main.constants import SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS, SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS_MONTHLY logger = logging.getLogger('awx.main.conf') @@ -805,6 +805,17 @@ register( category_slug='system', ) +register( + 'SUBSCRIPTION_USAGE_MODEL', + field_class=fields.ChoiceField, + choices=[SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS, SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS_MONTHLY], + default='', + allow_blank=True, + label=_('Defines subscription usage model and shows Host Metrics'), + category=_('System'), + category_slug='system', +) + def logging_validate(serializer, attrs): if not serializer.instance or not hasattr(serializer.instance, 'LOG_AGGREGATOR_HOST') or not hasattr(serializer.instance, 'LOG_AGGREGATOR_TYPE'): diff --git a/awx/main/constants.py b/awx/main/constants.py index 0271d70233..8f52b62aae 100644 --- a/awx/main/constants.py +++ b/awx/main/constants.py @@ -106,3 +106,7 @@ JOB_VARIABLE_PREFIXES = [ ANSIBLE_RUNNER_NEEDS_UPDATE_MESSAGE = ( '\u001b[31m \u001b[1m This can be caused if the version of ansible-runner in your execution environment is out of date.\u001b[0m' ) + +# Values for setting SUBSCRIPTION_USAGE_MODEL +SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS = 'unique_managed_hosts' +SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS_MONTHLY = 'unique_managed_hosts_monthly' diff --git a/awx/main/utils/licensing.py b/awx/main/utils/licensing.py index 3bc5e174e5..62c0deb56a 100644 --- a/awx/main/utils/licensing.py +++ b/awx/main/utils/licensing.py @@ -35,6 +35,7 @@ from cryptography import x509 from django.conf import settings from django.utils.translation import gettext_lazy as _ +from awx.main.constants import SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS MAX_INSTANCES = 9999999 @@ -382,8 +383,15 @@ class Licenser(object): current_instances = Host.objects.active_count() license_date = int(attrs.get('license_date', 0) or 0) - automated_instances = HostMetric.active_objects.count() - first_host = HostMetric.active_objects.only('first_automation').order_by('first_automation').first() + + model = getattr(settings, 'SUBSCRIPTION_USAGE_MODEL', '') + if model == SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS: + automated_instances = HostMetric.active_objects.count() + first_host = HostMetric.active_objects.only('first_automation').order_by('first_automation').first() + else: + automated_instances = HostMetric.objects.count() + first_host = HostMetric.objects.only('first_automation').order_by('first_automation').first() + if first_host: automated_since = int(first_host.first_automation.timestamp()) else: diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index b36dcc15ec..23d61a98a0 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -1028,3 +1028,9 @@ AWX_MOUNT_ISOLATED_PATHS_ON_K8S = False CLUSTER_HOST_ID = socket.gethostname() UI_NEXT = True + +# License compliance for total host count. Possible values: +# - '': No model - Subscription not counted from Host Metrics +# - 'unique_managed_hosts': Compliant = automated - deleted hosts (using /api/v2/host_metrics/) +# - 'unique_managed_hosts_monthly': TBD: AoC on Azure (now equal to '') +SUBSCRIPTION_USAGE_MODEL = '' From ae0c1730bb8860ab14ece76ac967eb4db94d35cb Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Wed, 8 Mar 2023 19:01:28 +0100 Subject: [PATCH 10/28] Subscription_usage_model in analytics/config.json --- awx/main/analytics/collectors.py | 3 ++- awx/main/conf.py | 10 ++++++++-- awx/main/constants.py | 1 - 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index 9ee363aed1..e7655997d2 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -83,7 +83,7 @@ def _identify_lower(key, since, until, last_gather): return lower, last_entries -@register('config', '1.4', description=_('General platform configuration.')) +@register('config', '1.5', description=_('General platform configuration.')) def config(since, **kwargs): license_info = get_license() install_type = 'traditional' @@ -119,6 +119,7 @@ def config(since, **kwargs): 'compliant': license_info.get('compliant'), 'date_warning': license_info.get('date_warning'), 'date_expired': license_info.get('date_expired'), + 'subscription_usage_model': getattr(settings, 'SUBSCRIPTION_USAGE_MODEL', ''), # 1.5+ 'free_instances': license_info.get('free_instances', 0), 'total_licensed_instances': license_info.get('instance_count', 0), 'license_expiry': license_info.get('time_remaining', 0), diff --git a/awx/main/conf.py b/awx/main/conf.py index 04dd056f45..f983b26a31 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -10,7 +10,7 @@ from rest_framework import serializers # AWX from awx.conf import fields, register, register_validate from awx.main.models import ExecutionEnvironment -from awx.main.constants import SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS, SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS_MONTHLY +from awx.main.constants import SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS logger = logging.getLogger('awx.main.conf') @@ -808,7 +808,13 @@ register( register( 'SUBSCRIPTION_USAGE_MODEL', field_class=fields.ChoiceField, - choices=[SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS, SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS_MONTHLY], + choices=[ + ('', _('Default model for AWX - no subscription')), + ( + SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS, + _('Usage based on unique managed nodes in a large historical time frame and delete functionality for no longer used managed nodes'), + ), + ], default='', allow_blank=True, label=_('Defines subscription usage model and shows Host Metrics'), diff --git a/awx/main/constants.py b/awx/main/constants.py index 8f52b62aae..85a14cca4c 100644 --- a/awx/main/constants.py +++ b/awx/main/constants.py @@ -109,4 +109,3 @@ ANSIBLE_RUNNER_NEEDS_UPDATE_MESSAGE = ( # Values for setting SUBSCRIPTION_USAGE_MODEL SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS = 'unique_managed_hosts' -SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS_MONTHLY = 'unique_managed_hosts_monthly' From 8d46d32944f539d4f5364aed626117520873ea7d Mon Sep 17 00:00:00 2001 From: Zita Nemeckova Date: Thu, 16 Feb 2023 12:26:48 +0100 Subject: [PATCH 11/28] UI --- awx/ui/src/api/index.js | 3 + awx/ui/src/api/models/HostMetrics.js | 10 +++ awx/ui/src/routeConfig.js | 6 ++ awx/ui/src/screens/HostMetrics/HostMetrics.js | 86 +++++++++++++++++++ .../HostMetrics/HostMetricsListItem.js | 27 ++++++ awx/ui/src/screens/HostMetrics/index.js | 1 + awx/ui/src/types.js | 9 ++ 7 files changed, 142 insertions(+) create mode 100644 awx/ui/src/api/models/HostMetrics.js create mode 100644 awx/ui/src/screens/HostMetrics/HostMetrics.js create mode 100644 awx/ui/src/screens/HostMetrics/HostMetricsListItem.js create mode 100644 awx/ui/src/screens/HostMetrics/index.js diff --git a/awx/ui/src/api/index.js b/awx/ui/src/api/index.js index 5281ad861d..7a03643c05 100644 --- a/awx/ui/src/api/index.js +++ b/awx/ui/src/api/index.js @@ -44,6 +44,7 @@ import WorkflowApprovalTemplates from './models/WorkflowApprovalTemplates'; import WorkflowJobTemplateNodes from './models/WorkflowJobTemplateNodes'; import WorkflowJobTemplates from './models/WorkflowJobTemplates'; import WorkflowJobs from './models/WorkflowJobs'; +import HostMetrics from './models/HostMetrics'; const ActivityStreamAPI = new ActivityStream(); const AdHocCommandsAPI = new AdHocCommands(); @@ -91,6 +92,7 @@ const WorkflowApprovalTemplatesAPI = new WorkflowApprovalTemplates(); const WorkflowJobTemplateNodesAPI = new WorkflowJobTemplateNodes(); const WorkflowJobTemplatesAPI = new WorkflowJobTemplates(); const WorkflowJobsAPI = new WorkflowJobs(); +const HostMetricsAPI = new HostMetrics(); export { ActivityStreamAPI, @@ -139,4 +141,5 @@ export { WorkflowJobTemplateNodesAPI, WorkflowJobTemplatesAPI, WorkflowJobsAPI, + HostMetricsAPI, }; diff --git a/awx/ui/src/api/models/HostMetrics.js b/awx/ui/src/api/models/HostMetrics.js new file mode 100644 index 0000000000..d8ca8f4c8c --- /dev/null +++ b/awx/ui/src/api/models/HostMetrics.js @@ -0,0 +1,10 @@ +import Base from '../Base'; + +class HostMetrics extends Base { + constructor(http) { + super(http); + this.baseUrl = 'api/v2/host_metrics/'; + } +} + +export default HostMetrics; diff --git a/awx/ui/src/routeConfig.js b/awx/ui/src/routeConfig.js index 602e804d2b..c738a352b3 100644 --- a/awx/ui/src/routeConfig.js +++ b/awx/ui/src/routeConfig.js @@ -23,6 +23,7 @@ import TopologyView from 'screens/TopologyView'; import Users from 'screens/User'; import WorkflowApprovals from 'screens/WorkflowApproval'; import { Jobs } from 'screens/Job'; +import HostMetrics from 'screens/HostMetrics'; function getRouteConfig(userProfile = {}) { let routeConfig = [ @@ -55,6 +56,11 @@ function getRouteConfig(userProfile = {}) { path: '/workflow_approvals', screen: WorkflowApprovals, }, + { + title: Host Metrics, + path: '/host_metrics', + screen: HostMetrics, + }, ], }, { diff --git a/awx/ui/src/screens/HostMetrics/HostMetrics.js b/awx/ui/src/screens/HostMetrics/HostMetrics.js new file mode 100644 index 0000000000..0a397dfce3 --- /dev/null +++ b/awx/ui/src/screens/HostMetrics/HostMetrics.js @@ -0,0 +1,86 @@ +import React, {useCallback, useEffect, useState} from 'react'; +import {t} from "@lingui/macro"; +import ScreenHeader from 'components/ScreenHeader/ScreenHeader'; +import { HostMetricsAPI } from 'api'; +import useRequest from 'hooks/useRequest'; +import PaginatedTable, { + HeaderRow, + HeaderCell +} from 'components/PaginatedTable'; +import DataListToolbar from 'components/DataListToolbar'; +import { getQSConfig, parseQueryString } from 'util/qs'; +import {Card, PageSection} from "@patternfly/react-core"; +import { useLocation } from 'react-router-dom'; +import HostMetricsListItem from "./HostMetricsListItem"; + +function HostMetrics() { + + const location = useLocation(); + + const [breadcrumbConfig] = useState({ + '/host_metrics': t`Host Metrics`, + }); + const QS_CONFIG = getQSConfig('host_metrics', { + page: 1, + page_size: 20, + order_by: 'hostname', + }); + const { + result: { count, results }, + isLoading, + error, + request: readHostMetrics, + } = useRequest( + useCallback(async () => { + const params = parseQueryString(QS_CONFIG, location.search); + const list = await HostMetricsAPI.read(params); + return { + count: list.data.count, + results: list.data.results + }; + }, [location.search]), + { results: [], count: 0 } + ); + + useEffect(() => { + readHostMetrics(); + }, [readHostMetrics]); + + return( + <> + + + + ()} + qsConfig={QS_CONFIG} + toolbarSearchColumns={[{name: t`Hostname`, key: 'hostname', isDefault: true}]} + toolbarSearchableKeys={[]} + toolbarRelatedSearchableKeys={[]} + renderToolbar={(props) => } + headerRow={ + + {t`Hostname`} + {t`First automated`} + {t`Last automated`} + {t`Automation`} + {t`Inventories`} + {t`Deleted`} + +} + /> + + + + ); +} + +export default HostMetrics; diff --git a/awx/ui/src/screens/HostMetrics/HostMetricsListItem.js b/awx/ui/src/screens/HostMetrics/HostMetricsListItem.js new file mode 100644 index 0000000000..fe9d06a25c --- /dev/null +++ b/awx/ui/src/screens/HostMetrics/HostMetricsListItem.js @@ -0,0 +1,27 @@ +import 'styled-components/macro'; +import React from 'react'; +import { Tr, Td } from '@patternfly/react-table'; +import { formatDateString } from 'util/dates'; +import { HostMetrics } from 'types'; +import {t} from "@lingui/macro"; + +function HostMetricsListItem({ item }) { + + return ( + + + {item.hostname} + {formatDateString(item.first_automation)} + {formatDateString(item.last_automation)} + {item.automated_counter} + {item.used_in_inventories || 0} + {item.deleted_counter}< + /Tr> + ); +} + +HostMetricsListItem.propTypes = { + item: HostMetrics.isRequired, +}; + +export default HostMetricsListItem; diff --git a/awx/ui/src/screens/HostMetrics/index.js b/awx/ui/src/screens/HostMetrics/index.js new file mode 100644 index 0000000000..bb2945686c --- /dev/null +++ b/awx/ui/src/screens/HostMetrics/index.js @@ -0,0 +1 @@ +export { default } from './HostMetrics'; diff --git a/awx/ui/src/types.js b/awx/ui/src/types.js index 57fd32029d..677f4edae8 100644 --- a/awx/ui/src/types.js +++ b/awx/ui/src/types.js @@ -439,3 +439,12 @@ export const Toast = shape({ hasTimeout: bool, message: string, }); + +export const HostMetrics = shape({ + hostname: string.isRequired, + first_automation: string.isRequired, + last_automation: string.isRequired, + automated_counter: number.isRequired, + used_in_inventories: number, + deleted_counter: number, +}); From 9135ff2f77c3f1918427848517f8f1e3683913a1 Mon Sep 17 00:00:00 2001 From: Zita Nemeckova Date: Mon, 20 Feb 2023 14:20:49 +0100 Subject: [PATCH 12/28] Add HostMetrics routes to the test --- awx/ui/src/routeConfig.test.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/awx/ui/src/routeConfig.test.js b/awx/ui/src/routeConfig.test.js index 0b84c670c3..cc73d96636 100644 --- a/awx/ui/src/routeConfig.test.js +++ b/awx/ui/src/routeConfig.test.js @@ -29,6 +29,7 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', + '/host_metrics', '/templates', '/credentials', '/projects', @@ -58,6 +59,7 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', + '/host_metrics', '/templates', '/credentials', '/projects', @@ -87,6 +89,7 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', + '/host_metrics', '/templates', '/credentials', '/projects', @@ -117,6 +120,7 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', + '/host_metrics', '/templates', '/credentials', '/projects', @@ -142,6 +146,7 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', + '/host_metrics', '/templates', '/credentials', '/projects', @@ -166,6 +171,7 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', + '/host_metrics', '/templates', '/credentials', '/projects', @@ -194,6 +200,7 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', + '/host_metrics', '/templates', '/credentials', '/projects', @@ -223,6 +230,7 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', + '/host_metrics', '/templates', '/credentials', '/projects', @@ -254,6 +262,7 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', + '/host_metrics', '/templates', '/credentials', '/projects', From d40fdd77ad312b71ebb1a35082e59beb8103cff6 Mon Sep 17 00:00:00 2001 From: Zita Nemeckova Date: Mon, 20 Feb 2023 15:21:49 +0100 Subject: [PATCH 13/28] Fix filter to take only hostname__icontains and disable advance search --- .../components/DataListToolbar/DataListToolbar.js | 12 ++++++++---- awx/ui/src/screens/HostMetrics/HostMetrics.js | 5 +++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/awx/ui/src/components/DataListToolbar/DataListToolbar.js b/awx/ui/src/components/DataListToolbar/DataListToolbar.js index 40deda10a1..9483989935 100644 --- a/awx/ui/src/components/DataListToolbar/DataListToolbar.js +++ b/awx/ui/src/components/DataListToolbar/DataListToolbar.js @@ -57,6 +57,7 @@ function DataListToolbar({ enableRelatedFuzzyFiltering, handleIsAnsibleFactsSelected, isFilterCleared, + advancedSearchDisabled, }) { const showExpandCollapse = onCompact && onExpand; const [isKebabOpen, setIsKebabOpen] = useState(false); @@ -86,6 +87,10 @@ function DataListToolbar({ }), [setIsKebabModalOpen] ); + const columns = [...searchColumns]; + if ( !advancedSearchDisabled ) { + columns.push({ name: t`Advanced`, key: 'advanced' }); + } return ( ()} qsConfig={QS_CONFIG} - toolbarSearchColumns={[{name: t`Hostname`, key: 'hostname', isDefault: true}]} + toolbarSearchColumns={[{name: t`Hostname`, key: 'hostname__icontains', isDefault: true}]} toolbarSearchableKeys={[]} toolbarRelatedSearchableKeys={[]} - renderToolbar={(props) => } + renderToolbar={(props) => {t`Hostname`} @@ -83,4 +83,5 @@ function HostMetrics() { ); } +export { HostMetrics as _HostMetrics }; export default HostMetrics; From 9f3c4f624024bd510426ed379b9af797e9a8d630 Mon Sep 17 00:00:00 2001 From: Zita Nemeckova Date: Wed, 22 Feb 2023 15:44:34 +0100 Subject: [PATCH 14/28] RBAC: only superuse and auditor can see HostMetrics --- awx/ui/src/routeConfig.js | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/ui/src/routeConfig.js b/awx/ui/src/routeConfig.js index c738a352b3..dae4643c5d 100644 --- a/awx/ui/src/routeConfig.js +++ b/awx/ui/src/routeConfig.js @@ -187,6 +187,7 @@ function getRouteConfig(userProfile = {}) { if (userProfile?.isSuperUser || userProfile?.isSystemAuditor) return routeConfig; + deleteRoute('host_metrics'); deleteRouteGroup('settings'); deleteRoute('management_jobs'); if (userProfile?.isOrgAdmin) return routeConfig; From 179868dff2f2d9fffb611584affbe293d786eba0 Mon Sep 17 00:00:00 2001 From: Zita Nemeckova Date: Thu, 23 Feb 2023 13:49:40 +0100 Subject: [PATCH 15/28] Add possibility to select and delete HostMetrics --- awx/ui/src/screens/HostMetrics/HostMetrics.js | 120 +++++++--- .../HostMetrics/HostMetricsDeleteButton.js | 220 ++++++++++++++++++ .../HostMetrics/HostMetricsListItem.js | 7 +- 3 files changed, 312 insertions(+), 35 deletions(-) create mode 100644 awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js diff --git a/awx/ui/src/screens/HostMetrics/HostMetrics.js b/awx/ui/src/screens/HostMetrics/HostMetrics.js index 7133bf7a18..ce43a04000 100644 --- a/awx/ui/src/screens/HostMetrics/HostMetrics.js +++ b/awx/ui/src/screens/HostMetrics/HostMetrics.js @@ -5,13 +5,16 @@ import { HostMetricsAPI } from 'api'; import useRequest from 'hooks/useRequest'; import PaginatedTable, { HeaderRow, - HeaderCell + HeaderCell, } from 'components/PaginatedTable'; import DataListToolbar from 'components/DataListToolbar'; import { getQSConfig, parseQueryString } from 'util/qs'; import {Card, PageSection} from "@patternfly/react-core"; import { useLocation } from 'react-router-dom'; import HostMetricsListItem from "./HostMetricsListItem"; +import HostMetricsDeleteButton from "./HostMetricsDeleteButton"; +import useSelected from 'hooks/useSelected'; + function HostMetrics() { @@ -46,40 +49,91 @@ function HostMetrics() { readHostMetrics(); }, [readHostMetrics]); + const { selected, isAllSelected, handleSelect, selectAll, clearSelected } = + useSelected(results); + return( - <> + <> - - - ()} - qsConfig={QS_CONFIG} - toolbarSearchColumns={[{name: t`Hostname`, key: 'hostname__icontains', isDefault: true}]} - toolbarSearchableKeys={[]} - toolbarRelatedSearchableKeys={[]} - renderToolbar={(props) => - {t`Hostname`} - {t`First automated`} - {t`Last automated`} - {t`Automation`} - {t`Inventories`} - {t`Deleted`} - -} - /> - - - + streamType="none" + breadcrumbConfig={breadcrumbConfig} + /> + + + ( + row.hostname === item.hostname)} + onSelect={() => handleSelect(item)} + rowIndex={index} + /> + )} + qsConfig={QS_CONFIG} + toolbarSearchColumns={[{name: t`Hostname`, key: 'hostname__icontains', isDefault: true}]} + toolbarSearchableKeys={[]} + toolbarRelatedSearchableKeys={[]} + renderToolbar={(props) => + + Promise.all(selected.map((hostMetric) => + HostMetricsAPI.destroy(hostMetric.id))) + .then(() => { readHostMetrics(); clearSelected(); }) + } + itemsToDelete={selected} + pluralizedItemName={t`Host Metrics`} + />]} + />} + headerRow={ + + + {t`Hostname`} + + + {t`First automated`} + + + {t`Last automated`} + + + {t`Automation`} + + + {t`Inventories`} + + + {t`Deleted`} + + + } + /> + + + ); } diff --git a/awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js b/awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js new file mode 100644 index 0000000000..9519ca519e --- /dev/null +++ b/awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js @@ -0,0 +1,220 @@ +import React, { useState } from 'react'; +import { + func, + node, + string, + arrayOf, + shape, +} from 'prop-types'; +import styled from 'styled-components'; +import { + Alert, + Badge, + Button, + Tooltip, +} from '@patternfly/react-core'; +import { t } from '@lingui/macro'; +import { getRelatedResourceDeleteCounts } from 'util/getRelatedResourceDeleteDetails'; +import AlertModal from '../../components/AlertModal'; + +import ErrorDetail from '../../components/ErrorDetail'; + +const WarningMessage = styled(Alert)` + margin-top: 10px; +`; + +const Label = styled.span` + && { + margin-right: 10px; + } +`; + +const ItemToDelete = shape({ + hostname: string.isRequired, +}); + +function HostMetricsDeleteButton({ + itemsToDelete, + pluralizedItemName, + errorMessage, + onDelete, + deleteDetailsRequests, + warningMessage, + deleteMessage, +}) { + const [isModalOpen, setIsModalOpen] = useState(false); + const [deleteDetails, setDeleteDetails] = useState(null); + const [isLoading, setIsLoading] = useState(false); + + const [deleteMessageError, setDeleteMessageError] = useState(); + const handleDelete = () => { + console.log("Delete"); + onDelete(); + toggleModal(); + }; + + const toggleModal = async (isOpen) => { + setIsLoading(true); + setDeleteDetails(null); + if ( + isOpen && + itemsToDelete.length === 1 && + deleteDetailsRequests?.length > 0 + ) { + const { results, error } = await getRelatedResourceDeleteCounts( + deleteDetailsRequests + ); + + if (error) { + setDeleteMessageError(error); + } else { + setDeleteDetails(results); + } + } + setIsLoading(false); + setIsModalOpen(isOpen); + }; + + const renderTooltip = () => { + if (itemsToDelete.length) { + return t`Soft delete`; + } else { + return t`Select a row to delete`; + } + }; + + const modalTitle = t`Soft delete ${pluralizedItemName}?`; + + const isDisabled = + itemsToDelete.length === 0; + + const buildDeleteWarning = () => { + const deleteMessages = []; + if (warningMessage) { + deleteMessages.push(warningMessage); + } + if (deleteMessage) { + if (itemsToDelete.length > 1 || deleteDetails) + { + deleteMessages.push(deleteMessage); + } else if (deleteDetails || itemsToDelete.length > 1) { + deleteMessages.push(deleteMessage); + } + } + return ( +
+ {deleteMessages.map((message) => ( +
+ {message} +
+ ))} + {deleteDetails && + Object.entries(deleteDetails).map(([key, value]) => ( +
+ + {value} +
+ ))} +
+ ); + }; + + if (deleteMessageError) { + return ( + { + toggleModal(false); + setDeleteMessageError(); + }} + > + + + ); + } + const shouldShowDeleteWarning = + warningMessage || + (itemsToDelete.length === 1 && deleteDetails) || + (itemsToDelete.length > 1 && deleteMessage); + + return ( + <> + +
+ +
+ + {isModalOpen && ( + toggleModal(false)} + actions={[ + , + , + ]} + > +
{t`This action will soft delete the following:`}
+ {itemsToDelete.map((item) => ( + + {item.hostname} +
+ + ))} + {shouldShowDeleteWarning && ( + + )} + + )} + + ); +} + +HostMetricsDeleteButton.propTypes = { + onDelete: func.isRequired, + itemsToDelete: arrayOf(ItemToDelete).isRequired, + pluralizedItemName: string, + warningMessage: node, +}; + +HostMetricsDeleteButton.defaultProps = { + pluralizedItemName: 'Items', + warningMessage: null, +}; + +export default HostMetricsDeleteButton; diff --git a/awx/ui/src/screens/HostMetrics/HostMetricsListItem.js b/awx/ui/src/screens/HostMetrics/HostMetricsListItem.js index fe9d06a25c..3fa4fb336e 100644 --- a/awx/ui/src/screens/HostMetrics/HostMetricsListItem.js +++ b/awx/ui/src/screens/HostMetrics/HostMetricsListItem.js @@ -4,12 +4,13 @@ import { Tr, Td } from '@patternfly/react-table'; import { formatDateString } from 'util/dates'; import { HostMetrics } from 'types'; import {t} from "@lingui/macro"; +import {bool, func} from "prop-types"; -function HostMetricsListItem({ item }) { +function HostMetricsListItem({ item, isSelected, onSelect, rowIndex }) { return ( - + {item.hostname} {formatDateString(item.first_automation)} {formatDateString(item.last_automation)} @@ -22,6 +23,8 @@ function HostMetricsListItem({ item }) { HostMetricsListItem.propTypes = { item: HostMetrics.isRequired, + isSelected: bool.isRequired, + onSelect: func.isRequired, }; export default HostMetricsListItem; From 610f75fcb1c5888b2d53826824fbcc47ca806055 Mon Sep 17 00:00:00 2001 From: Zita Nemeckova Date: Thu, 23 Feb 2023 15:47:28 +0100 Subject: [PATCH 16/28] Update routeConfig test to be according to RBAC --- awx/ui/src/routeConfig.test.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/awx/ui/src/routeConfig.test.js b/awx/ui/src/routeConfig.test.js index cc73d96636..4d2a237c47 100644 --- a/awx/ui/src/routeConfig.test.js +++ b/awx/ui/src/routeConfig.test.js @@ -89,7 +89,6 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', - '/host_metrics', '/templates', '/credentials', '/projects', @@ -120,7 +119,6 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', - '/host_metrics', '/templates', '/credentials', '/projects', @@ -146,7 +144,6 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', - '/host_metrics', '/templates', '/credentials', '/projects', @@ -171,7 +168,6 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', - '/host_metrics', '/templates', '/credentials', '/projects', @@ -200,7 +196,6 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', - '/host_metrics', '/templates', '/credentials', '/projects', @@ -230,7 +225,6 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', - '/host_metrics', '/templates', '/credentials', '/projects', @@ -262,7 +256,6 @@ describe('getRouteConfig', () => { '/schedules', '/activity_stream', '/workflow_approvals', - '/host_metrics', '/templates', '/credentials', '/projects', From 32a56311e61cc30ef05b3a0f72f9d9f62608f382 Mon Sep 17 00:00:00 2001 From: Zita Nemeckova Date: Tue, 28 Feb 2023 11:51:34 +0100 Subject: [PATCH 17/28] Fix linting issues --- awx/ui/src/screens/HostMetrics/HostMetrics.js | 17 +++-- .../screens/HostMetrics/HostMetrics.test.js | 68 +++++++++++++++++++ .../HostMetrics/HostMetricsDeleteButton.js | 12 +--- 3 files changed, 79 insertions(+), 18 deletions(-) create mode 100644 awx/ui/src/screens/HostMetrics/HostMetrics.test.js diff --git a/awx/ui/src/screens/HostMetrics/HostMetrics.js b/awx/ui/src/screens/HostMetrics/HostMetrics.js index ce43a04000..c85bbc6a33 100644 --- a/awx/ui/src/screens/HostMetrics/HostMetrics.js +++ b/awx/ui/src/screens/HostMetrics/HostMetrics.js @@ -11,23 +11,22 @@ import DataListToolbar from 'components/DataListToolbar'; import { getQSConfig, parseQueryString } from 'util/qs'; import {Card, PageSection} from "@patternfly/react-core"; import { useLocation } from 'react-router-dom'; +import useSelected from 'hooks/useSelected'; import HostMetricsListItem from "./HostMetricsListItem"; import HostMetricsDeleteButton from "./HostMetricsDeleteButton"; -import useSelected from 'hooks/useSelected'; +const QS_CONFIG = getQSConfig('host_metrics', { + page: 1, + page_size: 20, + order_by: 'hostname', +}); function HostMetrics() { - const location = useLocation(); const [breadcrumbConfig] = useState({ '/host_metrics': t`Host Metrics`, }); - const QS_CONFIG = getQSConfig('host_metrics', { - page: 1, - page_size: 20, - order_by: 'hostname', - }); const { result: { count, results }, isLoading, @@ -41,7 +40,7 @@ function HostMetrics() { count: list.data.count, results: list.data.results }; - }, [location.search]), + }, [location]), { results: [], count: 0 } ); @@ -81,7 +80,7 @@ function HostMetrics() { renderToolbar={(props) => el.find('ContentLoading').length === 0 + ); +} + +describe('', () => { + beforeEach(() => { + HostMetricsAPI.read.mockResolvedValue({ + data: { + count: mockHostMetrics.length, + results: mockHostMetrics, + }, + }); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('initially renders successfully', async () => { + await act(async () => { + mountWithContexts( + + ); + }); + }); + + test('HostMetrics are retrieved from the api and the components finishes loading', async () => { + let wrapper; + await act(async () => { + wrapper = mountWithContexts(); + }); + await waitForLoaded(wrapper); + + expect(HostMetricsAPI.read).toHaveBeenCalled(); + expect(wrapper.find('HostMetricsListItem')).toHaveLength(1); + }); +}); diff --git a/awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js b/awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js index 9519ca519e..46c8fcee14 100644 --- a/awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js +++ b/awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js @@ -36,7 +36,6 @@ const ItemToDelete = shape({ function HostMetricsDeleteButton({ itemsToDelete, pluralizedItemName, - errorMessage, onDelete, deleteDetailsRequests, warningMessage, @@ -48,7 +47,6 @@ function HostMetricsDeleteButton({ const [deleteMessageError, setDeleteMessageError] = useState(); const handleDelete = () => { - console.log("Delete"); onDelete(); toggleModal(); }; @@ -78,9 +76,8 @@ function HostMetricsDeleteButton({ const renderTooltip = () => { if (itemsToDelete.length) { return t`Soft delete`; - } else { - return t`Select a row to delete`; } + return t`Select a row to delete`; }; const modalTitle = t`Soft delete ${pluralizedItemName}?`; @@ -94,11 +91,8 @@ function HostMetricsDeleteButton({ deleteMessages.push(warningMessage); } if (deleteMessage) { - if (itemsToDelete.length > 1 || deleteDetails) - { - deleteMessages.push(deleteMessage); - } else if (deleteDetails || itemsToDelete.length > 1) { - deleteMessages.push(deleteMessage); + if (itemsToDelete.length > 1 || deleteDetails) { + deleteMessages.push(deleteMessage); } } return ( From 5be90fd36b7c023a1e8a5df78f966ae1486e29cf Mon Sep 17 00:00:00 2001 From: Zita Nemeckova Date: Tue, 28 Feb 2023 11:57:13 +0100 Subject: [PATCH 18/28] Do not show deleted host metrics --- awx/ui/src/screens/HostMetrics/HostMetrics.js | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/ui/src/screens/HostMetrics/HostMetrics.js b/awx/ui/src/screens/HostMetrics/HostMetrics.js index c85bbc6a33..04aa6ec5ef 100644 --- a/awx/ui/src/screens/HostMetrics/HostMetrics.js +++ b/awx/ui/src/screens/HostMetrics/HostMetrics.js @@ -19,6 +19,7 @@ const QS_CONFIG = getQSConfig('host_metrics', { page: 1, page_size: 20, order_by: 'hostname', + deleted: false, }); function HostMetrics() { From c20e8eb7120400e70924e01c7a07de5ab2aed98f Mon Sep 17 00:00:00 2001 From: Zita Nemeckova Date: Tue, 28 Feb 2023 12:08:06 +0100 Subject: [PATCH 19/28] Prettier --- .../DataListToolbar/DataListToolbar.js | 6 +- awx/ui/src/routeConfig.js | 10 +- awx/ui/src/screens/HostMetrics/HostMetrics.js | 148 ++++++++++-------- .../screens/HostMetrics/HostMetrics.test.js | 2 +- .../HostMetrics/HostMetricsDeleteButton.js | 55 +++---- .../HostMetrics/HostMetricsListItem.js | 30 ++-- 6 files changed, 131 insertions(+), 120 deletions(-) diff --git a/awx/ui/src/components/DataListToolbar/DataListToolbar.js b/awx/ui/src/components/DataListToolbar/DataListToolbar.js index 9483989935..39b3d6bb43 100644 --- a/awx/ui/src/components/DataListToolbar/DataListToolbar.js +++ b/awx/ui/src/components/DataListToolbar/DataListToolbar.js @@ -88,8 +88,8 @@ function DataListToolbar({ [setIsKebabModalOpen] ); const columns = [...searchColumns]; - if ( !advancedSearchDisabled ) { - columns.push({ name: t`Advanced`, key: 'advanced' }); + if (!advancedSearchDisabled) { + columns.push({ name: t`Advanced`, key: 'advanced' }); } return ( Host Metrics, - path: '/host_metrics', - screen: HostMetrics, - }, + { + title: Host Metrics, + path: '/host_metrics', + screen: HostMetrics, + }, ], }, { diff --git a/awx/ui/src/screens/HostMetrics/HostMetrics.js b/awx/ui/src/screens/HostMetrics/HostMetrics.js index 04aa6ec5ef..387ed03772 100644 --- a/awx/ui/src/screens/HostMetrics/HostMetrics.js +++ b/awx/ui/src/screens/HostMetrics/HostMetrics.js @@ -1,5 +1,5 @@ -import React, {useCallback, useEffect, useState} from 'react'; -import {t} from "@lingui/macro"; +import React, { useCallback, useEffect, useState } from 'react'; +import { t } from '@lingui/macro'; import ScreenHeader from 'components/ScreenHeader/ScreenHeader'; import { HostMetricsAPI } from 'api'; import useRequest from 'hooks/useRequest'; @@ -9,17 +9,17 @@ import PaginatedTable, { } from 'components/PaginatedTable'; import DataListToolbar from 'components/DataListToolbar'; import { getQSConfig, parseQueryString } from 'util/qs'; -import {Card, PageSection} from "@patternfly/react-core"; +import { Card, PageSection } from '@patternfly/react-core'; import { useLocation } from 'react-router-dom'; import useSelected from 'hooks/useSelected'; -import HostMetricsListItem from "./HostMetricsListItem"; -import HostMetricsDeleteButton from "./HostMetricsDeleteButton"; +import HostMetricsListItem from './HostMetricsListItem'; +import HostMetricsDeleteButton from './HostMetricsDeleteButton'; const QS_CONFIG = getQSConfig('host_metrics', { - page: 1, - page_size: 20, - order_by: 'hostname', - deleted: false, + page: 1, + page_size: 20, + order_by: 'hostname', + deleted: false, }); function HostMetrics() { @@ -34,15 +34,15 @@ function HostMetrics() { error, request: readHostMetrics, } = useRequest( - useCallback(async () => { - const params = parseQueryString(QS_CONFIG, location.search); - const list = await HostMetricsAPI.read(params); - return { - count: list.data.count, - results: list.data.results - }; - }, [location]), - { results: [], count: 0 } + useCallback(async () => { + const params = parseQueryString(QS_CONFIG, location.search); + const list = await HostMetricsAPI.read(params); + return { + count: list.data.count, + results: list.data.results, + }; + }, [location]), + { results: [], count: 0 } ); useEffect(() => { @@ -50,14 +50,11 @@ function HostMetrics() { }, [readHostMetrics]); const { selected, isAllSelected, handleSelect, selectAll, clearSelected } = - useSelected(results); + useSelected(results); - return( + return ( <> - + ( + renderRow={(item, index) => ( row.hostname === item.hostname)} + isSelected={selected.some( + (row) => row.hostname === item.hostname + )} onSelect={() => handleSelect(item)} rowIndex={index} /> )} qsConfig={QS_CONFIG} - toolbarSearchColumns={[{name: t`Hostname`, key: 'hostname__icontains', isDefault: true}]} + toolbarSearchColumns={[ + { + name: t`Hostname`, + key: 'hostname__icontains', + isDefault: true, + }, + ]} toolbarSearchableKeys={[]} toolbarRelatedSearchableKeys={[]} - renderToolbar={(props) => + renderToolbar={(props) => ( - Promise.all(selected.map((hostMetric) => - HostMetricsAPI.destroy(hostMetric.id))) - .then(() => { readHostMetrics(); clearSelected(); }) + Promise.all( + selected.map((hostMetric) => + HostMetricsAPI.destroy(hostMetric.id) + ) + ).then(() => { + readHostMetrics(); + clearSelected(); + }) } - itemsToDelete={selected} - pluralizedItemName={t`Host Metrics`} - />]} - />} - headerRow={ - - - {t`Hostname`} - - - {t`First automated`} - - - {t`Last automated`} - - - {t`Automation`} - - - {t`Inventories`} - - - {t`Deleted`} - - + itemsToDelete={selected} + pluralizedItemName={t`Host Metrics`} + />, + ]} + /> + )} + headerRow={ + + {t`Hostname`} + + {t`First automated`} + + + {t`Last automated`} + + + {t`Automation`} + + + {t`Inventories`} + + + {t`Deleted`} + + } /> diff --git a/awx/ui/src/screens/HostMetrics/HostMetrics.test.js b/awx/ui/src/screens/HostMetrics/HostMetrics.test.js index 884f25973b..972d216712 100644 --- a/awx/ui/src/screens/HostMetrics/HostMetrics.test.js +++ b/awx/ui/src/screens/HostMetrics/HostMetrics.test.js @@ -19,7 +19,7 @@ const mockHostMetrics = [ used_in_inventories: 1, deleted_counter: 1, id: 1, - } + }, ]; function waitForLoaded(wrapper) { diff --git a/awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js b/awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js index 46c8fcee14..a0b439e893 100644 --- a/awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js +++ b/awx/ui/src/screens/HostMetrics/HostMetricsDeleteButton.js @@ -1,18 +1,7 @@ import React, { useState } from 'react'; -import { - func, - node, - string, - arrayOf, - shape, -} from 'prop-types'; +import { func, node, string, arrayOf, shape } from 'prop-types'; import styled from 'styled-components'; -import { - Alert, - Badge, - Button, - Tooltip, -} from '@patternfly/react-core'; +import { Alert, Badge, Button, Tooltip } from '@patternfly/react-core'; import { t } from '@lingui/macro'; import { getRelatedResourceDeleteCounts } from 'util/getRelatedResourceDeleteDetails'; import AlertModal from '../../components/AlertModal'; @@ -82,8 +71,7 @@ function HostMetricsDeleteButton({ const modalTitle = t`Soft delete ${pluralizedItemName}?`; - const isDisabled = - itemsToDelete.length === 0; + const isDisabled = itemsToDelete.length === 0; const buildDeleteWarning = () => { const deleteMessages = []; @@ -92,7 +80,7 @@ function HostMetricsDeleteButton({ } if (deleteMessage) { if (itemsToDelete.length > 1 || deleteDetails) { - deleteMessages.push(deleteMessage); + deleteMessages.push(deleteMessage); } } return ( @@ -134,21 +122,21 @@ function HostMetricsDeleteButton({ return ( <> - -
- -
- + +
+ +
+ {isModalOpen && (
{t`This action will soft delete the following:`}
{itemsToDelete.map((item) => ( - + {item.hostname}
diff --git a/awx/ui/src/screens/HostMetrics/HostMetricsListItem.js b/awx/ui/src/screens/HostMetrics/HostMetricsListItem.js index 3fa4fb336e..10a73f54fa 100644 --- a/awx/ui/src/screens/HostMetrics/HostMetricsListItem.js +++ b/awx/ui/src/screens/HostMetrics/HostMetricsListItem.js @@ -3,28 +3,34 @@ import React from 'react'; import { Tr, Td } from '@patternfly/react-table'; import { formatDateString } from 'util/dates'; import { HostMetrics } from 'types'; -import {t} from "@lingui/macro"; -import {bool, func} from "prop-types"; +import { t } from '@lingui/macro'; +import { bool, func } from 'prop-types'; function HostMetricsListItem({ item, isSelected, onSelect, rowIndex }) { - return ( - - + + {item.hostname} - {formatDateString(item.first_automation)} - {formatDateString(item.last_automation)} + + {formatDateString(item.first_automation)} + + + {formatDateString(item.last_automation)} + {item.automated_counter} {item.used_in_inventories || 0} - {item.deleted_counter}< - /Tr> + {item.deleted_counter} + ); } HostMetricsListItem.propTypes = { - item: HostMetrics.isRequired, - isSelected: bool.isRequired, - onSelect: func.isRequired, + item: HostMetrics.isRequired, + isSelected: bool.isRequired, + onSelect: func.isRequired, }; export default HostMetricsListItem; From c117ca66d53bc272e75accfc270890490bc73fb3 Mon Sep 17 00:00:00 2001 From: Zita Nemeckova Date: Thu, 2 Mar 2023 14:28:02 +0100 Subject: [PATCH 20/28] Show HostMetrics only for specific subscription SUBSCRIPTION_USAGE_MODEL: 'unique_managed_hosts' Fixes https://issues.redhat.com/browse/AA-1613 --- awx/ui/src/api/models/Settings.js | 4 ++++ awx/ui/src/contexts/Config.js | 24 +++++++++++++++++++++--- awx/ui/src/routeConfig.js | 7 ++++++- awx/ui/src/routeConfig.test.js | 1 + 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/awx/ui/src/api/models/Settings.js b/awx/ui/src/api/models/Settings.js index 89aad94c00..10bab4c3a0 100644 --- a/awx/ui/src/api/models/Settings.js +++ b/awx/ui/src/api/models/Settings.js @@ -18,6 +18,10 @@ class Settings extends Base { return this.http.get(`${this.baseUrl}all/`); } + readSystem() { + return this.http.get(`${this.baseUrl}system/`); + } + updateCategory(category, data) { return this.http.patch(`${this.baseUrl}${category}/`, data); } diff --git a/awx/ui/src/contexts/Config.js b/awx/ui/src/contexts/Config.js index 1ef355ead3..fd28208b8b 100644 --- a/awx/ui/src/contexts/Config.js +++ b/awx/ui/src/contexts/Config.js @@ -8,6 +8,7 @@ import useRequest, { useDismissableError } from 'hooks/useRequest'; import AlertModal from 'components/AlertModal'; import ErrorDetail from 'components/ErrorDetail'; import { useSession } from './Session'; +import { SettingsAPI } from '../api'; // eslint-disable-next-line import/prefer-default-export export const ConfigContext = React.createContext({}); @@ -40,6 +41,11 @@ export const ConfigProvider = ({ children }) => { }, }, ] = await Promise.all([ConfigAPI.read(), MeAPI.read()]); + let systemConfig = {}; + if (me?.is_superuser || me?.is_system_auditor) { + const { data: systemConfigResults } = await SettingsAPI.readSystem(); + systemConfig = systemConfigResults; + } const [ { @@ -62,10 +68,21 @@ export const ConfigProvider = ({ children }) => { role_level: 'execution_environment_admin_role', }), ]); - - return { ...data, me, adminOrgCount, notifAdminCount, execEnvAdminCount }; + return { + ...data, + me, + adminOrgCount, + notifAdminCount, + execEnvAdminCount, + systemConfig, + }; }, []), - { adminOrgCount: 0, notifAdminCount: 0, execEnvAdminCount: 0 } + { + adminOrgCount: 0, + notifAdminCount: 0, + execEnvAdminCount: 0, + systemConfig: {}, + } ); const { error, dismissError } = useDismissableError(configError); @@ -112,6 +129,7 @@ export const useUserProfile = () => { isOrgAdmin: config.adminOrgCount, isNotificationAdmin: config.notifAdminCount, isExecEnvAdmin: config.execEnvAdminCount, + systemConfig: config.systemConfig, }; }; diff --git a/awx/ui/src/routeConfig.js b/awx/ui/src/routeConfig.js index 9e1cac0197..1accb1fa00 100644 --- a/awx/ui/src/routeConfig.js +++ b/awx/ui/src/routeConfig.js @@ -184,7 +184,12 @@ function getRouteConfig(userProfile = {}) { const deleteRouteGroup = (name) => { routeConfig = routeConfig.filter(({ groupId }) => !groupId.includes(name)); }; - + if ( + userProfile?.systemConfig?.SUBSCRIPTION_USAGE_MODEL !== + 'unique_managed_hosts' + ) { + deleteRoute('host_metrics'); + } if (userProfile?.isSuperUser || userProfile?.isSystemAuditor) return routeConfig; deleteRoute('host_metrics'); diff --git a/awx/ui/src/routeConfig.test.js b/awx/ui/src/routeConfig.test.js index 4d2a237c47..4888537485 100644 --- a/awx/ui/src/routeConfig.test.js +++ b/awx/ui/src/routeConfig.test.js @@ -7,6 +7,7 @@ const userProfile = { isOrgAdmin: false, isNotificationAdmin: false, isExecEnvAdmin: false, + systemConfig: { SUBSCRIPTION_USAGE_MODEL: 'unique_managed_hosts' }, }; const filterPaths = (sidebar) => { From 88bb6e5a6a01f3afcad002222674f5a68b365eec Mon Sep 17 00:00:00 2001 From: Zita Nemeckova Date: Mon, 6 Mar 2023 11:26:14 +0100 Subject: [PATCH 21/28] Fix test failure --- awx/ui/src/screens/HostMetrics/HostMetrics.js | 1 + awx/ui/src/screens/HostMetrics/HostMetrics.test.js | 1 + 2 files changed, 2 insertions(+) diff --git a/awx/ui/src/screens/HostMetrics/HostMetrics.js b/awx/ui/src/screens/HostMetrics/HostMetrics.js index 387ed03772..838bbf8127 100644 --- a/awx/ui/src/screens/HostMetrics/HostMetrics.js +++ b/awx/ui/src/screens/HostMetrics/HostMetrics.js @@ -65,6 +65,7 @@ function HostMetrics() { pluralizedItemName={t`Host Metrics`} renderRow={(item, index) => ( row.hostname === item.hostname diff --git a/awx/ui/src/screens/HostMetrics/HostMetrics.test.js b/awx/ui/src/screens/HostMetrics/HostMetrics.test.js index 972d216712..78d8c2bbdd 100644 --- a/awx/ui/src/screens/HostMetrics/HostMetrics.test.js +++ b/awx/ui/src/screens/HostMetrics/HostMetrics.test.js @@ -19,6 +19,7 @@ const mockHostMetrics = [ used_in_inventories: 1, deleted_counter: 1, id: 1, + url: '', }, ]; From 311cea5a4ac1c2eecd1c140da0e6170b634bb443 Mon Sep 17 00:00:00 2001 From: Aparna Karve Date: Tue, 14 Feb 2023 16:53:56 -0800 Subject: [PATCH 22/28] CLI for host usage collection --- awx/main/management/commands/host_metric.py | 214 +++++++++++++++++++- 1 file changed, 203 insertions(+), 11 deletions(-) diff --git a/awx/main/management/commands/host_metric.py b/awx/main/management/commands/host_metric.py index c58cea2c1b..a7be155e32 100644 --- a/awx/main/management/commands/host_metric.py +++ b/awx/main/management/commands/host_metric.py @@ -1,26 +1,196 @@ from django.core.management.base import BaseCommand import datetime from django.core.serializers.json import DjangoJSONEncoder -from awx.main.models.inventory import HostMetric +from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly +from awx.main.analytics.collectors import config +from awx.main.utils.encryption import get_encryption_key, Fernet256 +from django.utils.encoding import smart_str, smart_bytes +import base64 import json +import sys +import tempfile +import tarfile +import pandas as pd + +PREFERRED_ROW_COUNT = 500000 class Command(BaseCommand): help = 'This is for offline licensing usage' + def host_metric_queryset(self, result, offset=0, limit=PREFERRED_ROW_COUNT): + list_of_queryset = list( + result.values( + 'id', + 'hostname', + 'first_automation', + 'last_automation', + 'last_deleted', + 'automated_counter', + 'deleted_counter', + 'deleted', + 'used_in_inventories', + ).order_by('first_automation')[offset : offset + limit] + ) + + return list_of_queryset + + def host_metric_summary_monthly_queryset(self, result, offset=0, limit=PREFERRED_ROW_COUNT): + list_of_queryset = list( + result.values( + 'id', + 'date', + 'license_consumed', + 'license_capacity', + 'hosts_added', + 'hosts_deleted', + 'indirectly_managed_hosts', + ).order_by( + 'date' + )[offset : offset + limit] + ) + + return list_of_queryset + + def paginated_df(self, options, type, filter_kwargs, offset=0, limit=PREFERRED_ROW_COUNT): + list_of_queryset = [] + if type == 'host_metric': + result = HostMetric.objects.filter(**filter_kwargs) + list_of_queryset = self.host_metric_queryset(result, offset, limit) + elif type == 'host_metric_summary_monthly': + result = HostMetricSummaryMonthly.objects.filter(**filter_kwargs) + list_of_queryset = self.host_metric_summary_monthly_queryset(result, offset, limit) + + df = pd.DataFrame(list_of_queryset) + + if options['anonymized'] and 'hostname' in df.columns: + key = get_encryption_key('hostname', options.get('anonymized')) + df['hostname'] = df.apply(lambda x: self.obfuscated_hostname(key, x['hostname']), axis=1) + + return df + + def obfuscated_hostname(self, secret_sauce, hostname): + return self.encrypt_name(secret_sauce, hostname) + + def whole_page_count(self, row_count, rows_per_file): + whole_pages = int(row_count / rows_per_file) + partial_page = row_count % rows_per_file + if partial_page: + whole_pages += 1 + return whole_pages + + def csv_for_tar(self, options, temp_dir, type, filter_kwargs, index=1, offset=0, rows_per_file=PREFERRED_ROW_COUNT): + df = self.paginated_df(options, type, filter_kwargs, offset, rows_per_file) + csv_file = f'{temp_dir}/{type}{index}.csv' + arcname_file = f'{type}{index}.csv' + df.to_csv(csv_file, index=False) + return csv_file, arcname_file + + def config_for_tar(self, options, temp_dir): + config_json = json.dumps(config(options.get('since'))) + config_file = f'{temp_dir}/config.json' + arcname_file = 'config.json' + with open(config_file, 'w') as f: + f.write(config_json) + return config_file, arcname_file + + def encrypt_name(self, key, value): + value = smart_str(value) + f = Fernet256(key) + encrypted = f.encrypt(smart_bytes(value)) + b64data = smart_str(base64.b64encode(encrypted)) + tokens = ['$encrypted', 'UTF8', 'AESCBC', b64data] + return '$'.join(tokens) + + def decrypt_name(self, encryption_key, value): + raw_data = value[len('$encrypted$') :] + # If the encrypted string contains a UTF8 marker, discard it + utf8 = raw_data.startswith('UTF8$') + if utf8: + raw_data = raw_data[len('UTF8$') :] + algo, b64data = raw_data.split('$', 1) + if algo != 'AESCBC': + raise ValueError('unsupported algorithm: %s' % algo) + encrypted = base64.b64decode(b64data) + f = Fernet256(encryption_key) + value = f.decrypt(encrypted) + return smart_str(value) + + def output_json(self, options, filter_kwargs): + if not options.get('json') or options.get('json') == 'host_metric': + result = HostMetric.objects.filter(**filter_kwargs) + list_of_queryset = self.host_metric_queryset(result) + elif options.get('json') == 'host_metric_summary_monthly': + result = HostMetricSummaryMonthly.objects.filter(**filter_kwargs) + list_of_queryset = self.host_metric_summary_monthly_queryset(result) + + json_result = json.dumps(list_of_queryset, cls=DjangoJSONEncoder) + print(json_result) + + def output_csv(self, options, filter_kwargs): + with tempfile.TemporaryDirectory() as temp_dir: + if not options.get('csv') or options.get('csv') == 'host_metric': + csv_file, _arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric', filter_kwargs) + elif options.get('csv') == 'host_metric_summary_monthly': + csv_file, _arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric_summary_monthly', filter_kwargs) + with open(csv_file) as f: + sys.stdout.write(f.read()) + + def output_tarball(self, options, filter_kwargs, host_metric_row_count, host_metric_summary_monthly_row_count): + tar = tarfile.open("./host_metrics.tar.gz", "w:gz") + + with tempfile.TemporaryDirectory() as temp_dir: + if host_metric_row_count: + csv_file, arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric', filter_kwargs) + tar.add(csv_file, arcname=arcname_file) + + if host_metric_summary_monthly_row_count: + csv_file, arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric_summary_monthly', filter_kwargs) + tar.add(csv_file, arcname=arcname_file) + + config_file, arcname_file = self.config_for_tar(options, temp_dir) + tar.add(config_file, arcname=arcname_file) + + tar.close() + + def output_rows_per_file(self, options, filter_kwargs, host_metric_row_count, host_metric_summary_monthly_row_count): + rows_per_file = options.get('rows_per_file', PREFERRED_ROW_COUNT) + tar = tarfile.open("./host_metrics.tar.gz", "w:gz") + + host_metric_whole_pages = self.whole_page_count(host_metric_row_count, rows_per_file) + host_metric_summary_monthly_whole_pages = self.whole_page_count(host_metric_summary_monthly_row_count, rows_per_file) + + with tempfile.TemporaryDirectory() as temp_dir: + for index in range(host_metric_whole_pages): + offset = index * rows_per_file + + csv_file, arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric', filter_kwargs, index + 1, offset, rows_per_file) + tar.add(csv_file, arcname=arcname_file) + + for index in range(host_metric_summary_monthly_whole_pages): + offset = index * rows_per_file + + csv_file, arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric_summary_monthly', filter_kwargs, index + 1, offset, rows_per_file) + tar.add(csv_file, arcname=arcname_file) + + config_file, arcname_file = self.config_for_tar(options, temp_dir) + tar.add(config_file, arcname=arcname_file) + + tar.close() + def add_arguments(self, parser): parser.add_argument('--since', type=datetime.datetime.fromisoformat, help='Start Date in ISO format YYYY-MM-DD') parser.add_argument('--until', type=datetime.datetime.fromisoformat, help='End Date in ISO format YYYY-MM-DD') - parser.add_argument('--json', action='store_true', help='Select output as JSON') + parser.add_argument('--json', type=str, const='host_metric', nargs='?', help='Select output as JSON for host_metric or host_metric_summary_monthly') + parser.add_argument('--csv', type=str, const='host_metric', nargs='?', help='Select output as CSV for host_metric or host_metric_summary_monthly') + parser.add_argument('--tarball', action='store_true', help=f'Package CSV files into a tar with upto {PREFERRED_ROW_COUNT} rows') + parser.add_argument('--anonymized', type=str, help='Anonymize hostnames with provided salt') + parser.add_argument('--rows_per_file', type=int, help=f'Split rows in chunks of {PREFERRED_ROW_COUNT}') def handle(self, *args, **options): since = options.get('since') until = options.get('until') - if since is None and until is None: - print("No Arguments received") - return None - if since is not None and since.tzinfo is None: since = since.replace(tzinfo=datetime.timezone.utc) @@ -33,17 +203,39 @@ class Command(BaseCommand): if until is not None: filter_kwargs['last_automation__lte'] = until - result = HostMetric.objects.filter(**filter_kwargs) + filter_kwargs_host_metrics_summary = {} + if since is not None: + filter_kwargs_host_metrics_summary['date__gte'] = since + if until is not None: + filter_kwargs_host_metrics_summary['date__lte'] = until + + host_metric_row_count = HostMetric.objects.filter(**filter_kwargs).count() + host_metric_summary_monthly_row_count = HostMetricSummaryMonthly.objects.filter(**filter_kwargs_host_metrics_summary).count() + + if (host_metric_row_count > PREFERRED_ROW_COUNT or host_metric_summary_monthly_row_count > PREFERRED_ROW_COUNT) and ( + not options.get('rows_per_file') or options.get('rows_per_file') > PREFERRED_ROW_COUNT + ): + print( + f"HostMetric / HostMetricSummaryMonthly rows exceed the allowable limit of {PREFERRED_ROW_COUNT}. " + f"Set --rows_per_file {PREFERRED_ROW_COUNT} " + f"to split the rows in chunks of {PREFERRED_ROW_COUNT}" + ) + return # if --json flag is set, output the result in json format if options['json']: - list_of_queryset = list(result.values('hostname', 'first_automation', 'last_automation')) - json_result = json.dumps(list_of_queryset, cls=DjangoJSONEncoder) - print(json_result) + self.output_json(options, filter_kwargs) + elif options['csv']: + self.output_csv(options, filter_kwargs) + elif options['tarball']: + self.output_tarball(options, filter_kwargs, host_metric_row_count, host_metric_summary_monthly_row_count) + elif options['rows_per_file']: + self.output_rows_per_file(options, filter_kwargs, host_metric_row_count, host_metric_summary_monthly_row_count) # --json flag is not set, output in plain text else: - print(f"Total Number of hosts automated: {len(result)}") + print(f"Total Number of hosts automated: {host_metric_row_count}") + result = HostMetric.objects.filter(**filter_kwargs) for item in result: print( "Hostname : {hostname} | first_automation : {first_automation} | last_automation : {last_automation}".format( From 132fe5e443c1bf4cc9766a591625452409360f53 Mon Sep 17 00:00:00 2001 From: Aparna Karve Date: Thu, 23 Feb 2023 12:42:01 -0800 Subject: [PATCH 23/28] Remove `pandas` use `csv`. Also, remove anonymization --- awx/main/management/commands/host_metric.py | 51 +++++---------------- 1 file changed, 11 insertions(+), 40 deletions(-) diff --git a/awx/main/management/commands/host_metric.py b/awx/main/management/commands/host_metric.py index a7be155e32..5cf2aef18b 100644 --- a/awx/main/management/commands/host_metric.py +++ b/awx/main/management/commands/host_metric.py @@ -3,14 +3,11 @@ import datetime from django.core.serializers.json import DjangoJSONEncoder from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly from awx.main.analytics.collectors import config -from awx.main.utils.encryption import get_encryption_key, Fernet256 -from django.utils.encoding import smart_str, smart_bytes -import base64 import json import sys import tempfile import tarfile -import pandas as pd +import csv PREFERRED_ROW_COUNT = 500000 @@ -52,7 +49,7 @@ class Command(BaseCommand): return list_of_queryset - def paginated_df(self, options, type, filter_kwargs, offset=0, limit=PREFERRED_ROW_COUNT): + def paginated_db_retrieval(self, type, filter_kwargs, offset=0, limit=PREFERRED_ROW_COUNT): list_of_queryset = [] if type == 'host_metric': result = HostMetric.objects.filter(**filter_kwargs) @@ -61,16 +58,7 @@ class Command(BaseCommand): result = HostMetricSummaryMonthly.objects.filter(**filter_kwargs) list_of_queryset = self.host_metric_summary_monthly_queryset(result, offset, limit) - df = pd.DataFrame(list_of_queryset) - - if options['anonymized'] and 'hostname' in df.columns: - key = get_encryption_key('hostname', options.get('anonymized')) - df['hostname'] = df.apply(lambda x: self.obfuscated_hostname(key, x['hostname']), axis=1) - - return df - - def obfuscated_hostname(self, secret_sauce, hostname): - return self.encrypt_name(secret_sauce, hostname) + return list_of_queryset def whole_page_count(self, row_count, rows_per_file): whole_pages = int(row_count / rows_per_file) @@ -80,10 +68,16 @@ class Command(BaseCommand): return whole_pages def csv_for_tar(self, options, temp_dir, type, filter_kwargs, index=1, offset=0, rows_per_file=PREFERRED_ROW_COUNT): - df = self.paginated_df(options, type, filter_kwargs, offset, rows_per_file) + list_of_queryset = self.paginated_db_retrieval(type, filter_kwargs, offset, rows_per_file) csv_file = f'{temp_dir}/{type}{index}.csv' arcname_file = f'{type}{index}.csv' - df.to_csv(csv_file, index=False) + + with open(csv_file, 'w', newline='') as output_file: + keys = list_of_queryset[0].keys() if list_of_queryset else [] + dict_writer = csv.DictWriter(output_file, keys) + dict_writer.writeheader() + dict_writer.writerows(list_of_queryset) + return csv_file, arcname_file def config_for_tar(self, options, temp_dir): @@ -94,28 +88,6 @@ class Command(BaseCommand): f.write(config_json) return config_file, arcname_file - def encrypt_name(self, key, value): - value = smart_str(value) - f = Fernet256(key) - encrypted = f.encrypt(smart_bytes(value)) - b64data = smart_str(base64.b64encode(encrypted)) - tokens = ['$encrypted', 'UTF8', 'AESCBC', b64data] - return '$'.join(tokens) - - def decrypt_name(self, encryption_key, value): - raw_data = value[len('$encrypted$') :] - # If the encrypted string contains a UTF8 marker, discard it - utf8 = raw_data.startswith('UTF8$') - if utf8: - raw_data = raw_data[len('UTF8$') :] - algo, b64data = raw_data.split('$', 1) - if algo != 'AESCBC': - raise ValueError('unsupported algorithm: %s' % algo) - encrypted = base64.b64decode(b64data) - f = Fernet256(encryption_key) - value = f.decrypt(encrypted) - return smart_str(value) - def output_json(self, options, filter_kwargs): if not options.get('json') or options.get('json') == 'host_metric': result = HostMetric.objects.filter(**filter_kwargs) @@ -184,7 +156,6 @@ class Command(BaseCommand): parser.add_argument('--json', type=str, const='host_metric', nargs='?', help='Select output as JSON for host_metric or host_metric_summary_monthly') parser.add_argument('--csv', type=str, const='host_metric', nargs='?', help='Select output as CSV for host_metric or host_metric_summary_monthly') parser.add_argument('--tarball', action='store_true', help=f'Package CSV files into a tar with upto {PREFERRED_ROW_COUNT} rows') - parser.add_argument('--anonymized', type=str, help='Anonymize hostnames with provided salt') parser.add_argument('--rows_per_file', type=int, help=f'Split rows in chunks of {PREFERRED_ROW_COUNT}') def handle(self, *args, **options): From 878008a9c559fc8dadd0edaf053f4d492214dab9 Mon Sep 17 00:00:00 2001 From: Aparna Karve Date: Fri, 3 Mar 2023 14:20:22 -0800 Subject: [PATCH 24/28] make `rows_per_file` optional parameter Removed 2 sql statements that gave the info on row count which warranted many other changes --- awx/main/management/commands/host_metric.py | 145 ++++++++------------ 1 file changed, 58 insertions(+), 87 deletions(-) diff --git a/awx/main/management/commands/host_metric.py b/awx/main/management/commands/host_metric.py index 5cf2aef18b..5b38cb5fd5 100644 --- a/awx/main/management/commands/host_metric.py +++ b/awx/main/management/commands/host_metric.py @@ -49,36 +49,41 @@ class Command(BaseCommand): return list_of_queryset - def paginated_db_retrieval(self, type, filter_kwargs, offset=0, limit=PREFERRED_ROW_COUNT): + def paginated_db_retrieval(self, type, filter_kwargs, rows_per_file): + offset = 0 list_of_queryset = [] - if type == 'host_metric': - result = HostMetric.objects.filter(**filter_kwargs) - list_of_queryset = self.host_metric_queryset(result, offset, limit) - elif type == 'host_metric_summary_monthly': - result = HostMetricSummaryMonthly.objects.filter(**filter_kwargs) - list_of_queryset = self.host_metric_summary_monthly_queryset(result, offset, limit) + while True: + if type == 'host_metric': + result = HostMetric.objects.filter(**filter_kwargs) + list_of_queryset = self.host_metric_queryset(result, offset, rows_per_file) + elif type == 'host_metric_summary_monthly': + result = HostMetricSummaryMonthly.objects.filter(**filter_kwargs) + list_of_queryset = self.host_metric_summary_monthly_queryset(result, offset, rows_per_file) - return list_of_queryset + if not list_of_queryset: + break + else: + yield list_of_queryset - def whole_page_count(self, row_count, rows_per_file): - whole_pages = int(row_count / rows_per_file) - partial_page = row_count % rows_per_file - if partial_page: - whole_pages += 1 - return whole_pages + offset += len(list_of_queryset) - def csv_for_tar(self, options, temp_dir, type, filter_kwargs, index=1, offset=0, rows_per_file=PREFERRED_ROW_COUNT): - list_of_queryset = self.paginated_db_retrieval(type, filter_kwargs, offset, rows_per_file) - csv_file = f'{temp_dir}/{type}{index}.csv' - arcname_file = f'{type}{index}.csv' + def csv_for_tar(self, temp_dir, type, filter_kwargs, single_header=False, rows_per_file=PREFERRED_ROW_COUNT): + for index, list_of_queryset in enumerate(self.paginated_db_retrieval(type, filter_kwargs, rows_per_file)): + csv_file = f'{temp_dir}/{type}{index+1}.csv' + arcname_file = f'{type}{index+1}.csv' - with open(csv_file, 'w', newline='') as output_file: - keys = list_of_queryset[0].keys() if list_of_queryset else [] - dict_writer = csv.DictWriter(output_file, keys) - dict_writer.writeheader() - dict_writer.writerows(list_of_queryset) + with open(csv_file, 'w', newline='') as output_file: + try: + keys = list_of_queryset[0].keys() if list_of_queryset else [] + dict_writer = csv.DictWriter(output_file, keys) + if not single_header or index == 0: + dict_writer.writeheader() + dict_writer.writerows(list_of_queryset) - return csv_file, arcname_file + except Exception as e: + print(e) + + yield csv_file, arcname_file def config_for_tar(self, options, temp_dir): config_json = json.dumps(config(options.get('since'))) @@ -89,61 +94,37 @@ class Command(BaseCommand): return config_file, arcname_file def output_json(self, options, filter_kwargs): - if not options.get('json') or options.get('json') == 'host_metric': - result = HostMetric.objects.filter(**filter_kwargs) - list_of_queryset = self.host_metric_queryset(result) - elif options.get('json') == 'host_metric_summary_monthly': - result = HostMetricSummaryMonthly.objects.filter(**filter_kwargs) - list_of_queryset = self.host_metric_summary_monthly_queryset(result) + rows_per_file = options['rows_per_file'] or PREFERRED_ROW_COUNT + with tempfile.TemporaryDirectory() as temp_dir: + for csv_detail in self.csv_for_tar(temp_dir, options.get('json', 'host_metric'), filter_kwargs, False, rows_per_file): + csv_file = csv_detail[0] - json_result = json.dumps(list_of_queryset, cls=DjangoJSONEncoder) - print(json_result) + with open(csv_file) as f: + reader = csv.DictReader(f) + rows = list(reader) + json_result = json.dumps(rows, cls=DjangoJSONEncoder) + print(json_result) def output_csv(self, options, filter_kwargs): + rows_per_file = options['rows_per_file'] or PREFERRED_ROW_COUNT with tempfile.TemporaryDirectory() as temp_dir: - if not options.get('csv') or options.get('csv') == 'host_metric': - csv_file, _arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric', filter_kwargs) - elif options.get('csv') == 'host_metric_summary_monthly': - csv_file, _arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric_summary_monthly', filter_kwargs) - with open(csv_file) as f: - sys.stdout.write(f.read()) + for csv_detail in self.csv_for_tar(temp_dir, options.get('csv', 'host_metric'), filter_kwargs, True, rows_per_file): + csv_file = csv_detail[0] + with open(csv_file) as f: + sys.stdout.write(f.read()) + + def output_tarball(self, options, filter_kwargs): + single_header = False if options['rows_per_file'] else True + rows_per_file = options['rows_per_file'] or PREFERRED_ROW_COUNT - def output_tarball(self, options, filter_kwargs, host_metric_row_count, host_metric_summary_monthly_row_count): tar = tarfile.open("./host_metrics.tar.gz", "w:gz") with tempfile.TemporaryDirectory() as temp_dir: - if host_metric_row_count: - csv_file, arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric', filter_kwargs) - tar.add(csv_file, arcname=arcname_file) + for csv_detail in self.csv_for_tar(temp_dir, 'host_metric', filter_kwargs, single_header, rows_per_file): + tar.add(csv_detail[0], arcname=csv_detail[1]) - if host_metric_summary_monthly_row_count: - csv_file, arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric_summary_monthly', filter_kwargs) - tar.add(csv_file, arcname=arcname_file) - - config_file, arcname_file = self.config_for_tar(options, temp_dir) - tar.add(config_file, arcname=arcname_file) - - tar.close() - - def output_rows_per_file(self, options, filter_kwargs, host_metric_row_count, host_metric_summary_monthly_row_count): - rows_per_file = options.get('rows_per_file', PREFERRED_ROW_COUNT) - tar = tarfile.open("./host_metrics.tar.gz", "w:gz") - - host_metric_whole_pages = self.whole_page_count(host_metric_row_count, rows_per_file) - host_metric_summary_monthly_whole_pages = self.whole_page_count(host_metric_summary_monthly_row_count, rows_per_file) - - with tempfile.TemporaryDirectory() as temp_dir: - for index in range(host_metric_whole_pages): - offset = index * rows_per_file - - csv_file, arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric', filter_kwargs, index + 1, offset, rows_per_file) - tar.add(csv_file, arcname=arcname_file) - - for index in range(host_metric_summary_monthly_whole_pages): - offset = index * rows_per_file - - csv_file, arcname_file = self.csv_for_tar(options, temp_dir, 'host_metric_summary_monthly', filter_kwargs, index + 1, offset, rows_per_file) - tar.add(csv_file, arcname=arcname_file) + for csv_detail in self.csv_for_tar(temp_dir, 'host_metric_summary_monthly', filter_kwargs, single_header, rows_per_file): + tar.add(csv_detail[0], arcname=csv_detail[1]) config_file, arcname_file = self.config_for_tar(options, temp_dir) tar.add(config_file, arcname=arcname_file) @@ -180,17 +161,8 @@ class Command(BaseCommand): if until is not None: filter_kwargs_host_metrics_summary['date__lte'] = until - host_metric_row_count = HostMetric.objects.filter(**filter_kwargs).count() - host_metric_summary_monthly_row_count = HostMetricSummaryMonthly.objects.filter(**filter_kwargs_host_metrics_summary).count() - - if (host_metric_row_count > PREFERRED_ROW_COUNT or host_metric_summary_monthly_row_count > PREFERRED_ROW_COUNT) and ( - not options.get('rows_per_file') or options.get('rows_per_file') > PREFERRED_ROW_COUNT - ): - print( - f"HostMetric / HostMetricSummaryMonthly rows exceed the allowable limit of {PREFERRED_ROW_COUNT}. " - f"Set --rows_per_file {PREFERRED_ROW_COUNT} " - f"to split the rows in chunks of {PREFERRED_ROW_COUNT}" - ) + if options['rows_per_file'] and options.get('rows_per_file') > PREFERRED_ROW_COUNT: + print(f"rows_per_file exceeds the allowable limit of {PREFERRED_ROW_COUNT}.") return # if --json flag is set, output the result in json format @@ -199,18 +171,17 @@ class Command(BaseCommand): elif options['csv']: self.output_csv(options, filter_kwargs) elif options['tarball']: - self.output_tarball(options, filter_kwargs, host_metric_row_count, host_metric_summary_monthly_row_count) - elif options['rows_per_file']: - self.output_rows_per_file(options, filter_kwargs, host_metric_row_count, host_metric_summary_monthly_row_count) + self.output_tarball(options, filter_kwargs) # --json flag is not set, output in plain text else: - print(f"Total Number of hosts automated: {host_metric_row_count}") + print(f"Printing up to {PREFERRED_ROW_COUNT } automated hosts:") result = HostMetric.objects.filter(**filter_kwargs) - for item in result: + list_of_queryset = self.host_metric_queryset(result, 0, PREFERRED_ROW_COUNT) + for item in list_of_queryset: print( "Hostname : {hostname} | first_automation : {first_automation} | last_automation : {last_automation}".format( - hostname=item.hostname, first_automation=item.first_automation, last_automation=item.last_automation + hostname=item['hostname'], first_automation=item['first_automation'], last_automation=item['last_automation'] ) ) return From fbd5d79428751b9e30da51fbaecc5efb515350f4 Mon Sep 17 00:00:00 2001 From: Aparna Karve Date: Tue, 7 Mar 2023 13:49:24 -0800 Subject: [PATCH 25/28] Added internal batch processing for up to 10k rows For --rows_per_file if > 10k, rows would be fetched in batches of 10k --- awx/main/management/commands/host_metric.py | 106 +++++++++++++++----- 1 file changed, 79 insertions(+), 27 deletions(-) diff --git a/awx/main/management/commands/host_metric.py b/awx/main/management/commands/host_metric.py index 5b38cb5fd5..a0933b7fb9 100644 --- a/awx/main/management/commands/host_metric.py +++ b/awx/main/management/commands/host_metric.py @@ -9,13 +9,14 @@ import tempfile import tarfile import csv -PREFERRED_ROW_COUNT = 500000 +CSV_PREFERRED_ROW_COUNT = 500000 +BATCHED_FETCH_COUNT = 10000 class Command(BaseCommand): help = 'This is for offline licensing usage' - def host_metric_queryset(self, result, offset=0, limit=PREFERRED_ROW_COUNT): + def host_metric_queryset(self, result, offset=0, limit=BATCHED_FETCH_COUNT): list_of_queryset = list( result.values( 'id', @@ -32,7 +33,7 @@ class Command(BaseCommand): return list_of_queryset - def host_metric_summary_monthly_queryset(self, result, offset=0, limit=PREFERRED_ROW_COUNT): + def host_metric_summary_monthly_queryset(self, result, offset=0, limit=BATCHED_FETCH_COUNT): list_of_queryset = list( result.values( 'id', @@ -67,22 +68,70 @@ class Command(BaseCommand): offset += len(list_of_queryset) - def csv_for_tar(self, temp_dir, type, filter_kwargs, single_header=False, rows_per_file=PREFERRED_ROW_COUNT): + def controlled_db_retrieval(self, type, filter_kwargs, offset=0, fetch_count=BATCHED_FETCH_COUNT): + if type == 'host_metric': + result = HostMetric.objects.filter(**filter_kwargs) + return self.host_metric_queryset(result, offset, fetch_count) + elif type == 'host_metric_summary_monthly': + result = HostMetricSummaryMonthly.objects.filter(**filter_kwargs) + return self.host_metric_summary_monthly_queryset(result, offset, fetch_count) + + def write_to_csv(self, csv_file, list_of_queryset, always_header, first_write=False, mode='a'): + with open(csv_file, mode, newline='') as output_file: + try: + keys = list_of_queryset[0].keys() if list_of_queryset else [] + dict_writer = csv.DictWriter(output_file, keys) + if always_header or first_write: + dict_writer.writeheader() + dict_writer.writerows(list_of_queryset) + + except Exception as e: + print(e) + + def csv_for_tar(self, temp_dir, type, filter_kwargs, rows_per_file, always_header=True): for index, list_of_queryset in enumerate(self.paginated_db_retrieval(type, filter_kwargs, rows_per_file)): csv_file = f'{temp_dir}/{type}{index+1}.csv' arcname_file = f'{type}{index+1}.csv' - with open(csv_file, 'w', newline='') as output_file: - try: - keys = list_of_queryset[0].keys() if list_of_queryset else [] - dict_writer = csv.DictWriter(output_file, keys) - if not single_header or index == 0: - dict_writer.writeheader() - dict_writer.writerows(list_of_queryset) + first_write = True if index == 0 else False - except Exception as e: - print(e) + self.write_to_csv(csv_file, list_of_queryset, always_header, first_write, 'w') + yield csv_file, arcname_file + def csv_for_tar_batched_fetch(self, temp_dir, type, filter_kwargs, rows_per_file, always_header=True): + csv_iteration = 1 + + offset = 0 + rows_written_per_csv = 0 + to_fetch = BATCHED_FETCH_COUNT + + while True: + list_of_queryset = self.controlled_db_retrieval(type, filter_kwargs, offset, to_fetch) + + if not list_of_queryset: + break + + csv_file = f'{temp_dir}/{type}{csv_iteration}.csv' + arcname_file = f'{type}{csv_iteration}.csv' + self.write_to_csv(csv_file, list_of_queryset, always_header) + + offset += to_fetch + rows_written_per_csv += to_fetch + always_header = False + + remaining_rows_per_csv = rows_per_file - rows_written_per_csv + + if not remaining_rows_per_csv: + yield csv_file, arcname_file + + rows_written_per_csv = 0 + always_header = True + to_fetch = BATCHED_FETCH_COUNT + csv_iteration += 1 + elif remaining_rows_per_csv < BATCHED_FETCH_COUNT: + to_fetch = remaining_rows_per_csv + + if rows_written_per_csv: yield csv_file, arcname_file def config_for_tar(self, options, temp_dir): @@ -94,9 +143,8 @@ class Command(BaseCommand): return config_file, arcname_file def output_json(self, options, filter_kwargs): - rows_per_file = options['rows_per_file'] or PREFERRED_ROW_COUNT with tempfile.TemporaryDirectory() as temp_dir: - for csv_detail in self.csv_for_tar(temp_dir, options.get('json', 'host_metric'), filter_kwargs, False, rows_per_file): + for csv_detail in self.csv_for_tar(temp_dir, options.get('json', 'host_metric'), filter_kwargs, BATCHED_FETCH_COUNT, True): csv_file = csv_detail[0] with open(csv_file) as f: @@ -106,24 +154,28 @@ class Command(BaseCommand): print(json_result) def output_csv(self, options, filter_kwargs): - rows_per_file = options['rows_per_file'] or PREFERRED_ROW_COUNT with tempfile.TemporaryDirectory() as temp_dir: - for csv_detail in self.csv_for_tar(temp_dir, options.get('csv', 'host_metric'), filter_kwargs, True, rows_per_file): + for csv_detail in self.csv_for_tar(temp_dir, options.get('csv', 'host_metric'), filter_kwargs, BATCHED_FETCH_COUNT, False): csv_file = csv_detail[0] with open(csv_file) as f: sys.stdout.write(f.read()) def output_tarball(self, options, filter_kwargs): - single_header = False if options['rows_per_file'] else True - rows_per_file = options['rows_per_file'] or PREFERRED_ROW_COUNT + always_header = True + rows_per_file = options['rows_per_file'] or CSV_PREFERRED_ROW_COUNT tar = tarfile.open("./host_metrics.tar.gz", "w:gz") + if rows_per_file <= BATCHED_FETCH_COUNT: + csv_function = self.csv_for_tar + else: + csv_function = self.csv_for_tar_batched_fetch + with tempfile.TemporaryDirectory() as temp_dir: - for csv_detail in self.csv_for_tar(temp_dir, 'host_metric', filter_kwargs, single_header, rows_per_file): + for csv_detail in csv_function(temp_dir, 'host_metric', filter_kwargs, rows_per_file, always_header): tar.add(csv_detail[0], arcname=csv_detail[1]) - for csv_detail in self.csv_for_tar(temp_dir, 'host_metric_summary_monthly', filter_kwargs, single_header, rows_per_file): + for csv_detail in csv_function(temp_dir, 'host_metric_summary_monthly', filter_kwargs, rows_per_file, always_header): tar.add(csv_detail[0], arcname=csv_detail[1]) config_file, arcname_file = self.config_for_tar(options, temp_dir) @@ -136,8 +188,8 @@ class Command(BaseCommand): parser.add_argument('--until', type=datetime.datetime.fromisoformat, help='End Date in ISO format YYYY-MM-DD') parser.add_argument('--json', type=str, const='host_metric', nargs='?', help='Select output as JSON for host_metric or host_metric_summary_monthly') parser.add_argument('--csv', type=str, const='host_metric', nargs='?', help='Select output as CSV for host_metric or host_metric_summary_monthly') - parser.add_argument('--tarball', action='store_true', help=f'Package CSV files into a tar with upto {PREFERRED_ROW_COUNT} rows') - parser.add_argument('--rows_per_file', type=int, help=f'Split rows in chunks of {PREFERRED_ROW_COUNT}') + parser.add_argument('--tarball', action='store_true', help=f'Package CSV files into a tar with upto {CSV_PREFERRED_ROW_COUNT} rows') + parser.add_argument('--rows_per_file', type=int, help=f'Split rows in chunks of {CSV_PREFERRED_ROW_COUNT}') def handle(self, *args, **options): since = options.get('since') @@ -161,8 +213,8 @@ class Command(BaseCommand): if until is not None: filter_kwargs_host_metrics_summary['date__lte'] = until - if options['rows_per_file'] and options.get('rows_per_file') > PREFERRED_ROW_COUNT: - print(f"rows_per_file exceeds the allowable limit of {PREFERRED_ROW_COUNT}.") + if options['rows_per_file'] and options.get('rows_per_file') > CSV_PREFERRED_ROW_COUNT: + print(f"rows_per_file exceeds the allowable limit of {CSV_PREFERRED_ROW_COUNT}.") return # if --json flag is set, output the result in json format @@ -175,9 +227,9 @@ class Command(BaseCommand): # --json flag is not set, output in plain text else: - print(f"Printing up to {PREFERRED_ROW_COUNT } automated hosts:") + print(f"Printing up to {BATCHED_FETCH_COUNT } automated hosts:") result = HostMetric.objects.filter(**filter_kwargs) - list_of_queryset = self.host_metric_queryset(result, 0, PREFERRED_ROW_COUNT) + list_of_queryset = self.host_metric_queryset(result, 0, BATCHED_FETCH_COUNT) for item in list_of_queryset: print( "Hostname : {hostname} | first_automation : {first_automation} | last_automation : {last_automation}".format( From 382f98ceedd4f81a1692160f87358488d2a11d13 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Mon, 13 Mar 2023 22:14:56 -0400 Subject: [PATCH 26/28] Fixing migration files --- ...5_add_hostmetric_fields.py => 0180_add_hostmetric_fields.py} | 2 +- ...metricsummarymonthly.py => 0181_hostmetricsummarymonthly.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename awx/main/migrations/{0175_add_hostmetric_fields.py => 0180_add_hostmetric_fields.py} (96%) rename awx/main/migrations/{0176_hostmetricsummarymonthly.py => 0181_hostmetricsummarymonthly.py} (96%) diff --git a/awx/main/migrations/0175_add_hostmetric_fields.py b/awx/main/migrations/0180_add_hostmetric_fields.py similarity index 96% rename from awx/main/migrations/0175_add_hostmetric_fields.py rename to awx/main/migrations/0180_add_hostmetric_fields.py index 75090bd678..3d9048adb2 100644 --- a/awx/main/migrations/0175_add_hostmetric_fields.py +++ b/awx/main/migrations/0180_add_hostmetric_fields.py @@ -5,7 +5,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('main', '0174_ensure_org_ee_admin_roles'), + ('main', '0179_change_cyberark_plugin_names'), ] operations = [ diff --git a/awx/main/migrations/0176_hostmetricsummarymonthly.py b/awx/main/migrations/0181_hostmetricsummarymonthly.py similarity index 96% rename from awx/main/migrations/0176_hostmetricsummarymonthly.py rename to awx/main/migrations/0181_hostmetricsummarymonthly.py index fe482aa416..3dcac9c4c2 100644 --- a/awx/main/migrations/0176_hostmetricsummarymonthly.py +++ b/awx/main/migrations/0181_hostmetricsummarymonthly.py @@ -5,7 +5,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('main', '0175_add_hostmetric_fields'), + ('main', '0180_add_hostmetric_fields'), ] operations = [ From 8ec6e556a116d72c6d691848208b4422cfbbfb39 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Wed, 22 Mar 2023 11:31:12 +0100 Subject: [PATCH 27/28] HostMetricSummaryMonthly API commented out --- .../api/host_metric_summary_monthly_list.md | 12 ------------ awx/api/urls/urls.py | 5 +++-- awx/api/views/__init__.py | 19 ++++++++++--------- awx/api/views/root.py | 3 ++- awx/main/conf.py | 2 +- awx/main/management/commands/host_metric.py | 2 +- awx/main/utils/licensing.py | 4 ++-- awx/settings/defaults.py | 1 - 8 files changed, 19 insertions(+), 29 deletions(-) delete mode 100644 awx/api/templates/api/host_metric_summary_monthly_list.md diff --git a/awx/api/templates/api/host_metric_summary_monthly_list.md b/awx/api/templates/api/host_metric_summary_monthly_list.md deleted file mode 100644 index 953b1827a6..0000000000 --- a/awx/api/templates/api/host_metric_summary_monthly_list.md +++ /dev/null @@ -1,12 +0,0 @@ -# Intended Use Case - -To get summaries from a certain day or earlier, you can filter this -endpoint in the following way. - - ?date__gte=2023-01-01 - -This will return summaries that were produced on that date or later. -These host metric monthly summaries should be automatically produced -by a background task that runs once each month. - -{% include "api/list_api_view.md" %} diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index c7d73165c3..9eafb51d64 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -30,7 +30,7 @@ from awx.api.views import ( OAuth2TokenList, ApplicationOAuth2TokenList, OAuth2ApplicationDetail, - HostMetricSummaryMonthlyList, + # HostMetricSummaryMonthlyList, # It will be enabled in future version of the AWX ) from awx.api.views.bulk import ( @@ -121,7 +121,8 @@ v2_urls = [ re_path(r'^inventories/', include(inventory_urls)), re_path(r'^hosts/', include(host_urls)), re_path(r'^host_metrics/', include(host_metric_urls)), - re_path(r'^host_metric_summary_monthly/$', HostMetricSummaryMonthlyList.as_view(), name='host_metric_summary_monthly_list'), + # It will be enabled in future version of the AWX + # re_path(r'^host_metric_summary_monthly/$', HostMetricSummaryMonthlyList.as_view(), name='host_metric_summary_monthly_list'), re_path(r'^groups/', include(group_urls)), re_path(r'^inventory_sources/', include(inventory_source_urls)), re_path(r'^inventory_updates/', include(inventory_update_urls)), diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index a9f2e0ce25..c1e99a4002 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -1570,15 +1570,16 @@ class HostMetricDetail(RetrieveDestroyAPIView): return Response(status=status.HTTP_204_NO_CONTENT) -class HostMetricSummaryMonthlyList(ListAPIView): - name = _("Host Metrics Summary Monthly") - model = models.HostMetricSummaryMonthly - serializer_class = serializers.HostMetricSummaryMonthlySerializer - permission_classes = (IsSystemAdminOrAuditor,) - search_fields = ('date',) - - def get_queryset(self): - return self.model.objects.all() +# It will be enabled in future version of the AWX +# class HostMetricSummaryMonthlyList(ListAPIView): +# name = _("Host Metrics Summary Monthly") +# model = models.HostMetricSummaryMonthly +# serializer_class = serializers.HostMetricSummaryMonthlySerializer +# permission_classes = (IsSystemAdminOrAuditor,) +# search_fields = ('date',) +# +# def get_queryset(self): +# return self.model.objects.all() class HostList(HostRelatedSearchMixin, ListCreateAPIView): diff --git a/awx/api/views/root.py b/awx/api/views/root.py index f343d8169d..be4d9cc44b 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -103,7 +103,8 @@ class ApiVersionRootView(APIView): data['groups'] = reverse('api:group_list', request=request) data['hosts'] = reverse('api:host_list', request=request) data['host_metrics'] = reverse('api:host_metric_list', request=request) - data['host_metric_summary_monthly'] = reverse('api:host_metric_summary_monthly_list', request=request) + # It will be enabled in future version of the AWX + # data['host_metric_summary_monthly'] = reverse('api:host_metric_summary_monthly_list', request=request) data['job_templates'] = reverse('api:job_template_list', request=request) data['jobs'] = reverse('api:job_list', request=request) data['ad_hoc_commands'] = reverse('api:ad_hoc_command_list', request=request) diff --git a/awx/main/conf.py b/awx/main/conf.py index f983b26a31..33b3b4714d 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -809,7 +809,7 @@ register( 'SUBSCRIPTION_USAGE_MODEL', field_class=fields.ChoiceField, choices=[ - ('', _('Default model for AWX - no subscription')), + ('', _('Default model for AWX - no subscription. Deletion of host_metrics will not be considered for purposes of managed host counting')), ( SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS, _('Usage based on unique managed nodes in a large historical time frame and delete functionality for no longer used managed nodes'), diff --git a/awx/main/management/commands/host_metric.py b/awx/main/management/commands/host_metric.py index a0933b7fb9..1d76d634b7 100644 --- a/awx/main/management/commands/host_metric.py +++ b/awx/main/management/commands/host_metric.py @@ -227,7 +227,7 @@ class Command(BaseCommand): # --json flag is not set, output in plain text else: - print(f"Printing up to {BATCHED_FETCH_COUNT } automated hosts:") + print(f"Printing up to {BATCHED_FETCH_COUNT} automated hosts:") result = HostMetric.objects.filter(**filter_kwargs) list_of_queryset = self.host_metric_queryset(result, 0, BATCHED_FETCH_COUNT) for item in list_of_queryset: diff --git a/awx/main/utils/licensing.py b/awx/main/utils/licensing.py index 62c0deb56a..b3ea7723e5 100644 --- a/awx/main/utils/licensing.py +++ b/awx/main/utils/licensing.py @@ -384,8 +384,8 @@ class Licenser(object): current_instances = Host.objects.active_count() license_date = int(attrs.get('license_date', 0) or 0) - model = getattr(settings, 'SUBSCRIPTION_USAGE_MODEL', '') - if model == SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS: + subscription_model = getattr(settings, 'SUBSCRIPTION_USAGE_MODEL', '') + if subscription_model == SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS: automated_instances = HostMetric.active_objects.count() first_host = HostMetric.active_objects.only('first_automation').order_by('first_automation').first() else: diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 23d61a98a0..f3b6c18eef 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -1032,5 +1032,4 @@ UI_NEXT = True # License compliance for total host count. Possible values: # - '': No model - Subscription not counted from Host Metrics # - 'unique_managed_hosts': Compliant = automated - deleted hosts (using /api/v2/host_metrics/) -# - 'unique_managed_hosts_monthly': TBD: AoC on Azure (now equal to '') SUBSCRIPTION_USAGE_MODEL = '' From c30c9cbdbecc23d98d7b98c3207a32fba45f3c08 Mon Sep 17 00:00:00 2001 From: Aparna Karve Date: Wed, 22 Mar 2023 09:00:33 -0700 Subject: [PATCH 28/28] Remove --until option --- awx/main/management/commands/host_metric.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/awx/main/management/commands/host_metric.py b/awx/main/management/commands/host_metric.py index 1d76d634b7..c0862ea1a3 100644 --- a/awx/main/management/commands/host_metric.py +++ b/awx/main/management/commands/host_metric.py @@ -185,7 +185,6 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('--since', type=datetime.datetime.fromisoformat, help='Start Date in ISO format YYYY-MM-DD') - parser.add_argument('--until', type=datetime.datetime.fromisoformat, help='End Date in ISO format YYYY-MM-DD') parser.add_argument('--json', type=str, const='host_metric', nargs='?', help='Select output as JSON for host_metric or host_metric_summary_monthly') parser.add_argument('--csv', type=str, const='host_metric', nargs='?', help='Select output as CSV for host_metric or host_metric_summary_monthly') parser.add_argument('--tarball', action='store_true', help=f'Package CSV files into a tar with upto {CSV_PREFERRED_ROW_COUNT} rows') @@ -193,25 +192,17 @@ class Command(BaseCommand): def handle(self, *args, **options): since = options.get('since') - until = options.get('until') if since is not None and since.tzinfo is None: since = since.replace(tzinfo=datetime.timezone.utc) - if until is not None and until.tzinfo is None: - until = until.replace(tzinfo=datetime.timezone.utc) - filter_kwargs = {} if since is not None: filter_kwargs['last_automation__gte'] = since - if until is not None: - filter_kwargs['last_automation__lte'] = until filter_kwargs_host_metrics_summary = {} if since is not None: filter_kwargs_host_metrics_summary['date__gte'] = since - if until is not None: - filter_kwargs_host_metrics_summary['date__lte'] = until if options['rows_per_file'] and options.get('rows_per_file') > CSV_PREFERRED_ROW_COUNT: print(f"rows_per_file exceeds the allowable limit of {CSV_PREFERRED_ROW_COUNT}.")