diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index 5ecc30079d..c74f9f97e6 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -30,7 +30,7 @@ from awx.api.views import ( OAuth2TokenList, ApplicationOAuth2TokenList, OAuth2ApplicationDetail, - # HostMetricSummaryMonthlyList, # It will be enabled in future version of the AWX + HostMetricSummaryMonthlyList, ) from awx.api.views.bulk import ( @@ -123,8 +123,7 @@ v2_urls = [ re_path(r'^constructed_inventories/', include(constructed_inventory_urls)), re_path(r'^hosts/', include(host_urls)), re_path(r'^host_metrics/', include(host_metric_urls)), - # It will be enabled in future version of the AWX - # re_path(r'^host_metric_summary_monthly/$', HostMetricSummaryMonthlyList.as_view(), name='host_metric_summary_monthly_list'), + re_path(r'^host_metric_summary_monthly/$', HostMetricSummaryMonthlyList.as_view(), name='host_metric_summary_monthly_list'), re_path(r'^groups/', include(group_urls)), re_path(r'^inventory_sources/', include(inventory_source_urls)), re_path(r'^inventory_updates/', include(inventory_update_urls)), diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 5fdc0e29d7..893788268a 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -1564,16 +1564,15 @@ class HostMetricDetail(RetrieveDestroyAPIView): return Response(status=status.HTTP_204_NO_CONTENT) -# It will be enabled in future version of the AWX -# class HostMetricSummaryMonthlyList(ListAPIView): -# name = _("Host Metrics Summary Monthly") -# model = models.HostMetricSummaryMonthly -# serializer_class = serializers.HostMetricSummaryMonthlySerializer -# permission_classes = (IsSystemAdminOrAuditor,) -# search_fields = ('date',) -# -# def get_queryset(self): -# return self.model.objects.all() +class HostMetricSummaryMonthlyList(ListAPIView): + name = _("Host Metrics Summary Monthly") + model = models.HostMetricSummaryMonthly + serializer_class = serializers.HostMetricSummaryMonthlySerializer + permission_classes = (IsSystemAdminOrAuditor,) + search_fields = ('date',) + + def get_queryset(self): + return self.model.objects.all() class HostList(HostRelatedSearchMixin, ListCreateAPIView): diff --git a/awx/api/views/root.py b/awx/api/views/root.py index b076df41c7..3a9a910e1c 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -107,8 +107,7 @@ class ApiVersionRootView(APIView): data['groups'] = reverse('api:group_list', request=request) data['hosts'] = reverse('api:host_list', request=request) data['host_metrics'] = reverse('api:host_metric_list', request=request) - # It will be enabled in future version of the AWX - # data['host_metric_summary_monthly'] = reverse('api:host_metric_summary_monthly_list', request=request) + data['host_metric_summary_monthly'] = reverse('api:host_metric_summary_monthly_list', request=request) data['job_templates'] = reverse('api:job_template_list', request=request) data['jobs'] = reverse('api:job_list', request=request) data['ad_hoc_commands'] = reverse('api:ad_hoc_command_list', request=request) diff --git a/awx/main/conf.py b/awx/main/conf.py index cb47c22e06..767d5a251f 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -862,6 +862,15 @@ register( category_slug='system', ) +register( + 'HOST_METRIC_SUMMARY_TASK_LAST_TS', + field_class=fields.DateTimeField, + label=_('Last computing date of HostMetricSummaryMonthly'), + allow_null=True, + category=_('System'), + category_slug='system', +) + register( 'AWX_CLEANUP_PATHS', field_class=fields.BooleanField, diff --git a/awx/main/management/commands/cleanup_host_metrics.py b/awx/main/management/commands/cleanup_host_metrics.py index 788ffcba94..a250af9852 100644 --- a/awx/main/management/commands/cleanup_host_metrics.py +++ b/awx/main/management/commands/cleanup_host_metrics.py @@ -17,6 +17,6 @@ class Command(BaseCommand): months_ago = options.get('months-ago') or None if not months_ago: - months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_THRESHOLD', 12) + months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_SOFT_THRESHOLD', 12) HostMetric.cleanup_task(months_ago) diff --git a/awx/main/management/commands/host_metric_summary_monthly.py b/awx/main/management/commands/host_metric_summary_monthly.py new file mode 100644 index 0000000000..604380294a --- /dev/null +++ b/awx/main/management/commands/host_metric_summary_monthly.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand +from awx.main.tasks.host_metrics import HostMetricSummaryMonthlyTask + + +class Command(BaseCommand): + help = 'Computing of HostMetricSummaryMonthly' + + def handle(self, *args, **options): + HostMetricSummaryMonthlyTask().execute() diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index f92ff9e5fa..0310d3b1a2 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -899,18 +899,18 @@ class HostMetric(models.Model): last_automation_before = now() - dateutil.relativedelta.relativedelta(months=months_ago) - logger.info(f'Cleanup [HostMetric]: soft-deleting records last automated before {last_automation_before}') + logger.info(f'cleanup_host_metrics: soft-deleting records last automated before {last_automation_before}') HostMetric.active_objects.filter(last_automation__lt=last_automation_before).update( deleted=True, deleted_counter=models.F('deleted_counter') + 1, last_deleted=now() ) settings.CLEANUP_HOST_METRICS_LAST_TS = now() except (TypeError, ValueError): - logger.error(f"Cleanup [HostMetric]: months_ago({months_ago}) has to be a positive integer value") + logger.error(f"cleanup_host_metrics: months_ago({months_ago}) has to be a positive integer value") class HostMetricSummaryMonthly(models.Model): """ - HostMetric summaries computed by scheduled task monthly + HostMetric summaries computed by scheduled task 'awx.main.tasks.system.host_metric_summary_monthly' monthly """ date = models.DateField(unique=True) diff --git a/awx/main/tasks/__init__.py b/awx/main/tasks/__init__.py index 517df4a285..9794dd2b53 100644 --- a/awx/main/tasks/__init__.py +++ b/awx/main/tasks/__init__.py @@ -1 +1 @@ -from . import jobs, receptor, system # noqa +from . import host_metrics, jobs, receptor, system # noqa diff --git a/awx/main/tasks/host_metrics.py b/awx/main/tasks/host_metrics.py new file mode 100644 index 0000000000..abf658ef83 --- /dev/null +++ b/awx/main/tasks/host_metrics.py @@ -0,0 +1,205 @@ +import datetime +from dateutil.relativedelta import relativedelta +import logging + +from django.conf import settings +from django.db.models import Count +from django.db.models.functions import TruncMonth +from django.utils.timezone import now +from rest_framework.fields import DateTimeField +from awx.main.dispatch import get_task_queuename +from awx.main.dispatch.publish import task +from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly +from awx.conf.license import get_license + +logger = logging.getLogger('awx.main.tasks.host_metric_summary_monthly') + + +@task(queue=get_task_queuename) +def host_metric_summary_monthly(): + """Run cleanup host metrics summary monthly task each week""" + if _is_run_threshold_reached( + getattr(settings, 'HOST_METRIC_SUMMARY_TASK_LAST_TS', None), getattr(settings, 'HOST_METRIC_SUMMARY_TASK_INTERVAL', 7) * 86400 + ): + logger.info(f"Executing host_metric_summary_monthly, last ran at {getattr(settings, 'HOST_METRIC_SUMMARY_TASK_LAST_TS', '---')}") + HostMetricSummaryMonthlyTask().execute() + logger.info("Finished host_metric_summary_monthly") + + +def _is_run_threshold_reached(setting, threshold_seconds): + last_time = DateTimeField().to_internal_value(setting) if setting else DateTimeField().to_internal_value('1970-01-01') + + return (now() - last_time).total_seconds() > threshold_seconds + + +class HostMetricSummaryMonthlyTask: + """ + This task computes last [threshold] months of HostMetricSummaryMonthly table + [threshold] is setting CLEANUP_HOST_METRICS_HARD_THRESHOLD + Each record in the table represents changes in HostMetric table in one month + It always overrides all the months newer than , never updates older months + Algorithm: + - hosts_added are HostMetric records with first_automation in given month + - hosts_deleted are HostMetric records with deleted=True and last_deleted in given month + - - HostMetrics soft-deleted before also increases hosts_deleted in their last_deleted month + - license_consumed is license_consumed(previous month) + hosts_added - hosts_deleted + - - license_consumed for HostMetricSummaryMonthly.date < [threshold] is computed also from + all HostMetrics.first_automation < [threshold] + - license_capacity is set only for current month, and it's never updated (value taken from current subscription) + """ + + def __init__(self): + self.host_metrics = {} + self.processed_month = self._get_first_month() + self.existing_summaries = None + self.existing_summaries_idx = 0 + self.existing_summaries_cnt = 0 + self.records_to_create = [] + self.records_to_update = [] + + def execute(self): + self._load_existing_summaries() + self._load_hosts_added() + self._load_hosts_deleted() + + # Get first month after last hard delete + month = self._get_first_month() + license_consumed = self._get_license_consumed_before(month) + + # Fill record for each month + while month <= datetime.date.today().replace(day=1): + summary = self._find_or_create_summary(month) + # Update summary and update license_consumed by hosts added/removed this month + self._update_summary(summary, month, license_consumed) + license_consumed = summary.license_consumed + + month = month + relativedelta(months=1) + + # Create/Update stats + HostMetricSummaryMonthly.objects.bulk_create(self.records_to_create, batch_size=1000) + HostMetricSummaryMonthly.objects.bulk_update(self.records_to_update, ['license_consumed', 'hosts_added', 'hosts_deleted'], batch_size=1000) + + # Set timestamp of last run + settings.HOST_METRIC_SUMMARY_TASK_LAST_TS = now() + + def _get_license_consumed_before(self, month): + license_consumed = 0 + for metric_month, metric in self.host_metrics.items(): + if metric_month < month: + hosts_added = metric.get('hosts_added', 0) + hosts_deleted = metric.get('hosts_deleted', 0) + license_consumed = license_consumed + hosts_added - hosts_deleted + else: + break + return license_consumed + + def _load_existing_summaries(self): + """Find all summaries newer than host metrics delete threshold""" + self.existing_summaries = HostMetricSummaryMonthly.objects.filter(date__gte=self._get_first_month()).order_by('date') + self.existing_summaries_idx = 0 + self.existing_summaries_cnt = len(self.existing_summaries) + + def _load_hosts_added(self): + """Aggregates hosts added each month, by the 'first_automation' timestamp""" + # + # -- SQL translation (for better code readability) + # SELECT date_trunc('month', first_automation) as month, + # count(first_automation) AS hosts_added + # FROM main_hostmetric + # GROUP BY month + # ORDER by month; + result = ( + HostMetric.objects.annotate(month=TruncMonth('first_automation')) + .values('month') + .annotate(hosts_added=Count('first_automation')) + .values('month', 'hosts_added') + .order_by('month') + ) + + for host_metric in list(result): + month = host_metric['month'] + if month: + beginning_of_month = datetime.date(month.year, month.month, 1) + if self.host_metrics.get(beginning_of_month) is None: + self.host_metrics[beginning_of_month] = {} + self.host_metrics[beginning_of_month]['hosts_added'] = host_metric['hosts_added'] + + def _load_hosts_deleted(self): + """ + Aggregates hosts deleted each month, by the 'last_deleted' timestamp. + Host metrics have to be deleted NOW to be counted as deleted before + (by intention - statistics can change retrospectively by re-automation of previously deleted host) + """ + # + # -- SQL translation (for better code readability) + # SELECT date_trunc('month', last_deleted) as month, + # count(last_deleted) AS hosts_deleted + # FROM main_hostmetric + # WHERE deleted = True + # GROUP BY 1 # equal to "GROUP BY month" + # ORDER by month; + result = ( + HostMetric.objects.annotate(month=TruncMonth('last_deleted')) + .values('month') + .annotate(hosts_deleted=Count('last_deleted')) + .values('month', 'hosts_deleted') + .filter(deleted=True) + .order_by('month') + ) + for host_metric in list(result): + month = host_metric['month'] + if month: + beginning_of_month = datetime.date(month.year, month.month, 1) + if self.host_metrics.get(beginning_of_month) is None: + self.host_metrics[beginning_of_month] = {} + self.host_metrics[beginning_of_month]['hosts_deleted'] = host_metric['hosts_deleted'] + + def _find_or_create_summary(self, month): + summary = self._find_summary(month) + + if not summary: + summary = HostMetricSummaryMonthly(date=month) + self.records_to_create.append(summary) + else: + self.records_to_update.append(summary) + return summary + + def _find_summary(self, month): + """ + Existing summaries are ordered by month ASC. + This method is called with month in ascending order too => only 1 traversing is enough + """ + summary = None + while not summary and self.existing_summaries_idx < self.existing_summaries_cnt: + tmp = self.existing_summaries[self.existing_summaries_idx] + if tmp.date < month: + self.existing_summaries_idx += 1 + elif tmp.date == month: + summary = tmp + elif tmp.date > month: + break + return summary + + def _update_summary(self, summary, month, license_consumed): + """Updates the metric with hosts added and deleted and set license info for current month""" + # Get month counts from host metrics, zero if not found + hosts_added, hosts_deleted = 0, 0 + if metric := self.host_metrics.get(month, None): + hosts_added = metric.get('hosts_added', 0) + hosts_deleted = metric.get('hosts_deleted', 0) + + summary.license_consumed = license_consumed + hosts_added - hosts_deleted + summary.hosts_added = hosts_added + summary.hosts_deleted = hosts_deleted + + # Set subscription count for current month + if month == datetime.date.today().replace(day=1): + license_info = get_license() + summary.license_capacity = license_info.get('instance_count', 0) + return summary + + @staticmethod + def _get_first_month(): + """Returns first month after host metrics hard delete threshold""" + threshold = getattr(settings, 'CLEANUP_HOST_METRICS_HARD_THRESHOLD', 36) + return datetime.date.today().replace(day=1) - relativedelta(months=int(threshold) - 1) diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index 6fc42d371b..ed49d2c0af 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -316,13 +316,8 @@ def send_notifications(notification_list, job_id=None): @task(queue=get_task_queuename) def gather_analytics(): from awx.conf.models import Setting - from rest_framework.fields import DateTimeField - last_gather = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_GATHER').first() - last_time = DateTimeField().to_internal_value(last_gather.value) if last_gather and last_gather.value else None - gather_time = now() - - if not last_time or ((gather_time - last_time).total_seconds() > settings.AUTOMATION_ANALYTICS_GATHER_INTERVAL): + if is_run_threshold_reached(Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_GATHER').first(), settings.AUTOMATION_ANALYTICS_GATHER_INTERVAL): analytics.gather() @@ -381,16 +376,25 @@ def cleanup_images_and_files(): @task(queue=get_task_queuename) def cleanup_host_metrics(): + """Run cleanup host metrics ~each month""" + # TODO: move whole method to host_metrics in follow-up PR from awx.conf.models import Setting + + if is_run_threshold_reached( + Setting.objects.filter(key='CLEANUP_HOST_METRICS_LAST_TS').first(), getattr(settings, 'CLEANUP_HOST_METRICS_INTERVAL', 30) * 86400 + ): + months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_SOFT_THRESHOLD', 12) + logger.info("Executing cleanup_host_metrics") + HostMetric.cleanup_task(months_ago) + logger.info("Finished cleanup_host_metrics") + + +def is_run_threshold_reached(setting, threshold_seconds): from rest_framework.fields import DateTimeField - last_cleanup = Setting.objects.filter(key='CLEANUP_HOST_METRICS_LAST_TS').first() - last_time = DateTimeField().to_internal_value(last_cleanup.value) if last_cleanup and last_cleanup.value else None + last_time = DateTimeField().to_internal_value(setting.value) if setting and setting.value else DateTimeField().to_internal_value('1970-01-01') - cleanup_interval_secs = getattr(settings, 'CLEANUP_HOST_METRICS_INTERVAL', 30) * 86400 - if not last_time or ((now() - last_time).total_seconds() > cleanup_interval_secs): - months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_THRESHOLD', 12) - HostMetric.cleanup_task(months_ago) + return (now() - last_time).total_seconds() > threshold_seconds @task(queue=get_task_queuename) diff --git a/awx/main/tests/factories/fixtures.py b/awx/main/tests/factories/fixtures.py index 27556d6efe..9f4229718d 100644 --- a/awx/main/tests/factories/fixtures.py +++ b/awx/main/tests/factories/fixtures.py @@ -1,6 +1,9 @@ import json from django.contrib.auth.models import User +from django.core.exceptions import ValidationError + +from unittest import mock from awx.main.models import ( Organization, @@ -20,6 +23,7 @@ from awx.main.models import ( WorkflowJobNode, WorkflowJobTemplateNode, ) +from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly # mk methods should create only a single object of a single type. # they should also have the option of being persisted or not. @@ -248,3 +252,42 @@ def mk_workflow_job_node(unified_job_template=None, success_nodes=None, failure_ if persisted: workflow_node.save() return workflow_node + + +def mk_host_metric(hostname, first_automation, last_automation=None, last_deleted=None, deleted=False, persisted=True): + ok, idx = False, 1 + while not ok: + try: + with mock.patch("django.utils.timezone.now") as mock_now: + mock_now.return_value = first_automation + metric = HostMetric( + hostname=hostname or f"host-{first_automation}-{idx}", + first_automation=first_automation, + last_automation=last_automation or first_automation, + last_deleted=last_deleted, + deleted=deleted, + ) + metric.validate_unique() + if persisted: + metric.save() + ok = True + except ValidationError as e: + # Repeat create for auto-generated hostname + if not hostname and e.message_dict.get('hostname', None): + idx += 1 + else: + raise e + + +def mk_host_metric_summary(date, license_consumed=0, license_capacity=0, hosts_added=0, hosts_deleted=0, indirectly_managed_hosts=0, persisted=True): + summary = HostMetricSummaryMonthly( + date=date, + license_consumed=license_consumed, + license_capacity=license_capacity, + hosts_added=hosts_added, + hosts_deleted=hosts_deleted, + indirectly_managed_hosts=indirectly_managed_hosts, + ) + if persisted: + summary.save() + return summary diff --git a/awx/main/tests/functional/commands/test_host_metric_summary_monthly.py b/awx/main/tests/functional/commands/test_host_metric_summary_monthly.py new file mode 100644 index 0000000000..525fdd789f --- /dev/null +++ b/awx/main/tests/functional/commands/test_host_metric_summary_monthly.py @@ -0,0 +1,382 @@ +import pytest +import datetime +from dateutil.relativedelta import relativedelta +from django.conf import settings +from django.utils import timezone + + +from awx.main.management.commands.host_metric_summary_monthly import Command +from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly +from awx.main.tests.factories.fixtures import mk_host_metric, mk_host_metric_summary + + +@pytest.fixture +def threshold(): + return int(getattr(settings, 'CLEANUP_HOST_METRICS_HARD_THRESHOLD', 36)) + + +@pytest.mark.django_db +@pytest.mark.parametrize("metrics_cnt", [0, 1, 2, 3]) +@pytest.mark.parametrize("mode", ["old_data", "actual_data", "all_data"]) +def test_summaries_counts(threshold, metrics_cnt, mode): + assert HostMetricSummaryMonthly.objects.count() == 0 + + for idx in range(metrics_cnt): + if mode == "old_data" or mode == "all_data": + mk_host_metric(None, months_ago(threshold + idx, "dt")) + elif mode == "actual_data" or mode == "all_data": + mk_host_metric(None, (months_ago(threshold - idx, "dt"))) + + Command().handle() + + # Number of records is equal to host metrics' hard cleanup months + assert HostMetricSummaryMonthly.objects.count() == threshold + + # Records start with date in the month following to the threshold month + date = months_ago(threshold - 1) + for metric in list(HostMetricSummaryMonthly.objects.order_by('date').all()): + assert metric.date == date + date += relativedelta(months=1) + + # Older record are untouched + mk_host_metric_summary(date=months_ago(threshold + 10)) + Command().handle() + + assert HostMetricSummaryMonthly.objects.count() == threshold + 1 + + +@pytest.mark.django_db +@pytest.mark.parametrize("mode", ["old_data", "actual_data", "all_data"]) +def test_summary_values(threshold, mode): + tester = {"old_data": MetricsTesterOldData(threshold), "actual_data": MetricsTesterActualData(threshold), "all_data": MetricsTesterCombinedData(threshold)}[ + mode + ] + + for iteration in ["create_metrics", "add_old_summaries", "change_metrics", "delete_metrics", "add_metrics"]: + getattr(tester, iteration)() # call method by string + + # Operation is idempotent, repeat twice + for _ in range(2): + Command().handle() + # call assert method by string + getattr(tester, f"assert_{iteration}")() + + +class MetricsTester: + def __init__(self, threshold, ignore_asserts=False): + self.threshold = threshold + self.expected_summaries = {} + self.ignore_asserts = ignore_asserts + + def add_old_summaries(self): + """These records don't correspond with Host metrics""" + mk_host_metric_summary(self.below(4), license_consumed=100, hosts_added=10, hosts_deleted=5) + mk_host_metric_summary(self.below(3), license_consumed=105, hosts_added=20, hosts_deleted=10) + mk_host_metric_summary(self.below(2), license_consumed=115, hosts_added=60, hosts_deleted=75) + + def assert_add_old_summaries(self): + """Old summary records should be untouched""" + self.expected_summaries[self.below(4)] = {"date": self.below(4), "license_consumed": 100, "hosts_added": 10, "hosts_deleted": 5} + self.expected_summaries[self.below(3)] = {"date": self.below(3), "license_consumed": 105, "hosts_added": 20, "hosts_deleted": 10} + self.expected_summaries[self.below(2)] = {"date": self.below(2), "license_consumed": 115, "hosts_added": 60, "hosts_deleted": 75} + + self.assert_host_metric_summaries() + + def assert_host_metric_summaries(self): + """Ignore asserts when old/actual test object is used only as a helper for Combined test""" + if self.ignore_asserts: + return True + + for summary in list(HostMetricSummaryMonthly.objects.order_by('date').all()): + assert self.expected_summaries.get(summary.date, None) is not None + + assert self.expected_summaries[summary.date] == { + "date": summary.date, + "license_consumed": summary.license_consumed, + "hosts_added": summary.hosts_added, + "hosts_deleted": summary.hosts_deleted, + } + + def below(self, months, fmt="date"): + """months below threshold, returns first date of that month""" + date = months_ago(self.threshold + months) + if fmt == "dt": + return timezone.make_aware(datetime.datetime.combine(date, datetime.datetime.min.time())) + else: + return date + + def above(self, months, fmt="date"): + """months above threshold, returns first date of that month""" + date = months_ago(self.threshold - months) + if fmt == "dt": + return timezone.make_aware(datetime.datetime.combine(date, datetime.datetime.min.time())) + else: + return date + + +class MetricsTesterOldData(MetricsTester): + def create_metrics(self): + """Creates 7 host metrics older than delete threshold""" + mk_host_metric("host_1", first_automation=self.below(3, "dt")) + mk_host_metric("host_2", first_automation=self.below(2, "dt")) + mk_host_metric("host_3", first_automation=self.below(2, "dt"), last_deleted=self.above(2, "dt"), deleted=False) + mk_host_metric("host_4", first_automation=self.below(2, "dt"), last_deleted=self.above(2, "dt"), deleted=True) + mk_host_metric("host_5", first_automation=self.below(2, "dt"), last_deleted=self.below(2, "dt"), deleted=True) + mk_host_metric("host_6", first_automation=self.below(1, "dt"), last_deleted=self.below(1, "dt"), deleted=False) + mk_host_metric("host_7", first_automation=self.below(1, "dt")) + + def assert_create_metrics(self): + """ + Month 1 is computed from older host metrics, + Month 2 has deletion (host_4) + Other months are unchanged (same as month 2) + """ + self.expected_summaries = { + self.above(1): {"date": self.above(1), "license_consumed": 6, "hosts_added": 0, "hosts_deleted": 0}, + self.above(2): {"date": self.above(2), "license_consumed": 5, "hosts_added": 0, "hosts_deleted": 1}, + } + # no change in months 3+ + idx = 3 + month = self.above(idx) + while month <= beginning_of_the_month(): + self.expected_summaries[self.above(idx)] = {"date": self.above(idx), "license_consumed": 5, "hosts_added": 0, "hosts_deleted": 0} + month += relativedelta(months=1) + idx += 1 + + self.assert_host_metric_summaries() + + def add_old_summaries(self): + super().add_old_summaries() + + def assert_add_old_summaries(self): + super().assert_add_old_summaries() + + @staticmethod + def change_metrics(): + """Hosts 1,2 soft deleted, host_4 automated again (undeleted)""" + HostMetric.objects.filter(hostname='host_1').update(last_deleted=beginning_of_the_month("dt"), deleted=True) + HostMetric.objects.filter(hostname='host_2').update(last_deleted=timezone.now(), deleted=True) + HostMetric.objects.filter(hostname='host_4').update(deleted=False) + + def assert_change_metrics(self): + """ + Summaries since month 2 were changed (host_4 restored == automated again) + Current month has 2 deletions (host_1, host_2) + """ + self.expected_summaries[self.above(2)] |= {'hosts_deleted': 0} + for idx in range(2, self.threshold): + self.expected_summaries[self.above(idx)] |= {'license_consumed': 6} + self.expected_summaries[beginning_of_the_month()] |= {'license_consumed': 4, 'hosts_deleted': 2} + + self.assert_host_metric_summaries() + + @staticmethod + def delete_metrics(): + """Deletes metric deleted before the threshold""" + HostMetric.objects.filter(hostname='host_5').delete() + + def assert_delete_metrics(self): + """No change""" + self.assert_host_metric_summaries() + + @staticmethod + def add_metrics(): + """Adds new metrics""" + mk_host_metric("host_24", first_automation=beginning_of_the_month("dt")) + mk_host_metric("host_25", first_automation=beginning_of_the_month("dt")) # timezone.now()) + + def assert_add_metrics(self): + """Summary in current month is updated""" + self.expected_summaries[beginning_of_the_month()]['license_consumed'] = 6 + self.expected_summaries[beginning_of_the_month()]['hosts_added'] = 2 + + self.assert_host_metric_summaries() + + +class MetricsTesterActualData(MetricsTester): + def create_metrics(self): + """Creates 16 host metrics newer than delete threshold""" + mk_host_metric("host_8", first_automation=self.above(1, "dt")) + mk_host_metric("host_9", first_automation=self.above(1, "dt"), last_deleted=self.above(1, "dt")) + mk_host_metric("host_10", first_automation=self.above(1, "dt"), last_deleted=self.above(1, "dt"), deleted=True) + mk_host_metric("host_11", first_automation=self.above(1, "dt"), last_deleted=self.above(2, "dt")) + mk_host_metric("host_12", first_automation=self.above(1, "dt"), last_deleted=self.above(2, "dt"), deleted=True) + mk_host_metric("host_13", first_automation=self.above(2, "dt")) + mk_host_metric("host_14", first_automation=self.above(2, "dt"), last_deleted=self.above(2, "dt")) + mk_host_metric("host_15", first_automation=self.above(2, "dt"), last_deleted=self.above(2, "dt"), deleted=True) + mk_host_metric("host_16", first_automation=self.above(2, "dt"), last_deleted=self.above(3, "dt")) + mk_host_metric("host_17", first_automation=self.above(2, "dt"), last_deleted=self.above(3, "dt"), deleted=True) + mk_host_metric("host_18", first_automation=self.above(4, "dt")) + # next one shouldn't happen in real (deleted=True, last_deleted = NULL) + mk_host_metric("host_19", first_automation=self.above(4, "dt"), deleted=True) + mk_host_metric("host_20", first_automation=self.above(4, "dt"), last_deleted=self.above(4, "dt")) + mk_host_metric("host_21", first_automation=self.above(4, "dt"), last_deleted=self.above(4, "dt"), deleted=True) + mk_host_metric("host_22", first_automation=self.above(4, "dt"), last_deleted=self.above(5, "dt")) + mk_host_metric("host_23", first_automation=self.above(4, "dt"), last_deleted=self.above(5, "dt"), deleted=True) + + def assert_create_metrics(self): + self.expected_summaries = { + self.above(1): {"date": self.above(1), "license_consumed": 4, "hosts_added": 5, "hosts_deleted": 1}, + self.above(2): {"date": self.above(2), "license_consumed": 7, "hosts_added": 5, "hosts_deleted": 2}, + self.above(3): {"date": self.above(3), "license_consumed": 6, "hosts_added": 0, "hosts_deleted": 1}, + self.above(4): {"date": self.above(4), "license_consumed": 11, "hosts_added": 6, "hosts_deleted": 1}, + self.above(5): {"date": self.above(5), "license_consumed": 10, "hosts_added": 0, "hosts_deleted": 1}, + } + # no change in months 6+ + idx = 6 + month = self.above(idx) + while month <= beginning_of_the_month(): + self.expected_summaries[self.above(idx)] = {"date": self.above(idx), "license_consumed": 10, "hosts_added": 0, "hosts_deleted": 0} + month += relativedelta(months=1) + idx += 1 + + self.assert_host_metric_summaries() + + def add_old_summaries(self): + super().add_old_summaries() + + def assert_add_old_summaries(self): + super().assert_add_old_summaries() + + @staticmethod + def change_metrics(): + """ + - Hosts 12, 19, 21 were automated again (undeleted) + - Host 16 was soft deleted + - Host 17 was undeleted and soft deleted again + """ + HostMetric.objects.filter(hostname='host_12').update(deleted=False) + HostMetric.objects.filter(hostname='host_16').update(last_deleted=timezone.now(), deleted=True) + HostMetric.objects.filter(hostname='host_17').update(last_deleted=beginning_of_the_month("dt"), deleted=True) + HostMetric.objects.filter(hostname='host_19').update(deleted=False) + HostMetric.objects.filter(hostname='host_21').update(deleted=False) + + def assert_change_metrics(self): + """ + Summaries since month 2 were changed + Current month has 2 deletions (host_16, host_17) + """ + self.expected_summaries[self.above(2)] |= {'license_consumed': 8, 'hosts_deleted': 1} + self.expected_summaries[self.above(3)] |= {'license_consumed': 8, 'hosts_deleted': 0} + self.expected_summaries[self.above(4)] |= {'license_consumed': 14, 'hosts_deleted': 0} + + # month 5 had hosts_deleted 1 => license_consumed == 14 - 1 + for idx in range(5, self.threshold): + self.expected_summaries[self.above(idx)] |= {'license_consumed': 13} + self.expected_summaries[beginning_of_the_month()] |= {'license_consumed': 11, 'hosts_deleted': 2} + + self.assert_host_metric_summaries() + + def delete_metrics(self): + """Hard cleanup can't delete metrics newer than threshold. No change""" + pass + + def assert_delete_metrics(self): + """No change""" + self.assert_host_metric_summaries() + + @staticmethod + def add_metrics(): + """Adds new metrics""" + mk_host_metric("host_26", first_automation=beginning_of_the_month("dt")) + mk_host_metric("host_27", first_automation=timezone.now()) + + def assert_add_metrics(self): + """ + Two metrics were deleted in current month by change_metrics() + Two metrics are added now + => license_consumed is equal to the previous month (13 - 2 + 2) + """ + self.expected_summaries[beginning_of_the_month()] |= {'license_consumed': 13, 'hosts_added': 2} + + self.assert_host_metric_summaries() + + +class MetricsTesterCombinedData(MetricsTester): + def __init__(self, threshold): + super().__init__(threshold) + self.old_data = MetricsTesterOldData(threshold, ignore_asserts=True) + self.actual_data = MetricsTesterActualData(threshold, ignore_asserts=True) + + def assert_host_metric_summaries(self): + self._combine_expected_summaries() + super().assert_host_metric_summaries() + + def create_metrics(self): + self.old_data.create_metrics() + self.actual_data.create_metrics() + + def assert_create_metrics(self): + self.old_data.assert_create_metrics() + self.actual_data.assert_create_metrics() + + self.assert_host_metric_summaries() + + def add_old_summaries(self): + super().add_old_summaries() + + def assert_add_old_summaries(self): + self.old_data.assert_add_old_summaries() + self.actual_data.assert_add_old_summaries() + + self.assert_host_metric_summaries() + + def change_metrics(self): + self.old_data.change_metrics() + self.actual_data.change_metrics() + + def assert_change_metrics(self): + self.old_data.assert_change_metrics() + self.actual_data.assert_change_metrics() + + self.assert_host_metric_summaries() + + def delete_metrics(self): + self.old_data.delete_metrics() + self.actual_data.delete_metrics() + + def assert_delete_metrics(self): + self.old_data.assert_delete_metrics() + self.actual_data.assert_delete_metrics() + + self.assert_host_metric_summaries() + + def add_metrics(self): + self.old_data.add_metrics() + self.actual_data.add_metrics() + + def assert_add_metrics(self): + self.old_data.assert_add_metrics() + self.actual_data.assert_add_metrics() + + self.assert_host_metric_summaries() + + def _combine_expected_summaries(self): + """ + Expected summaries are sum of expected values for tests with old and actual data + Except data older than hard delete threshold (these summaries are untouched by task => the same in all tests) + """ + for date, summary in self.old_data.expected_summaries.items(): + if date <= months_ago(self.threshold): + license_consumed = summary['license_consumed'] + hosts_added = summary['hosts_added'] + hosts_deleted = summary['hosts_deleted'] + else: + license_consumed = summary['license_consumed'] + self.actual_data.expected_summaries[date]['license_consumed'] + hosts_added = summary['hosts_added'] + self.actual_data.expected_summaries[date]['hosts_added'] + hosts_deleted = summary['hosts_deleted'] + self.actual_data.expected_summaries[date]['hosts_deleted'] + self.expected_summaries[date] = {'date': date, 'license_consumed': license_consumed, 'hosts_added': hosts_added, 'hosts_deleted': hosts_deleted} + + +def months_ago(num, fmt="date"): + if num is None: + return None + return beginning_of_the_month(fmt) - relativedelta(months=num) + + +def beginning_of_the_month(fmt="date"): + date = datetime.date.today().replace(day=1) + if fmt == "dt": + return timezone.make_aware(datetime.datetime.combine(date, datetime.datetime.min.time())) + else: + return date diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index ba82132ad4..714aa6ea6f 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -471,7 +471,8 @@ CELERYBEAT_SCHEDULE = { 'receptor_reaper': {'task': 'awx.main.tasks.system.awx_receptor_workunit_reaper', 'schedule': timedelta(seconds=60)}, 'send_subsystem_metrics': {'task': 'awx.main.analytics.analytics_tasks.send_subsystem_metrics', 'schedule': timedelta(seconds=20)}, 'cleanup_images': {'task': 'awx.main.tasks.system.cleanup_images_and_files', 'schedule': timedelta(hours=3)}, - 'cleanup_host_metrics': {'task': 'awx.main.tasks.system.cleanup_host_metrics', 'schedule': timedelta(days=1)}, + 'cleanup_host_metrics': {'task': 'awx.main.tasks.system.cleanup_host_metrics', 'schedule': timedelta(hours=3, minutes=30)}, + 'host_metric_summary_monthly': {'task': 'awx.main.tasks.host_metrics.host_metric_summary_monthly', 'schedule': timedelta(hours=4)}, } # Django Caching Configuration @@ -1054,4 +1055,12 @@ CLEANUP_HOST_METRICS_LAST_TS = None # Host metrics cleanup - minimal interval between two cleanups in days CLEANUP_HOST_METRICS_INTERVAL = 30 # days # Host metrics cleanup - soft-delete HostMetric records with last_automation < [threshold] (in months) -CLEANUP_HOST_METRICS_THRESHOLD = 12 # months +CLEANUP_HOST_METRICS_SOFT_THRESHOLD = 12 # months +# Host metrics cleanup +# - delete HostMetric record with deleted=True and last_deleted < [threshold] +# - also threshold for computing HostMetricSummaryMonthly (command/scheduled task) +CLEANUP_HOST_METRICS_HARD_THRESHOLD = 36 # months + +# Host metric summary monthly task - last time of run +HOST_METRIC_SUMMARY_TASK_LAST_TS = None +HOST_METRIC_SUMMARY_TASK_INTERVAL = 7 # days