mirror of
https://github.com/ansible/awx.git
synced 2026-01-11 01:57:35 -03:30
HostMetricSummaryMonthly command + views + scheduled task (#13999)
Co-authored-by: Alan Rominger <arominge@redhat.com>
This commit is contained in:
Martin Slemr
GitHub
parent
0edcd688a2
commit
6c5590e0e6
@ -30,7 +30,7 @@ from awx.api.views import (
|
||||
OAuth2TokenList,
|
||||
ApplicationOAuth2TokenList,
|
||||
OAuth2ApplicationDetail,
|
||||
# HostMetricSummaryMonthlyList, # It will be enabled in future version of the AWX
|
||||
HostMetricSummaryMonthlyList,
|
||||
)
|
||||
|
||||
from awx.api.views.bulk import (
|
||||
@ -123,8 +123,7 @@ v2_urls = [
|
||||
re_path(r'^constructed_inventories/', include(constructed_inventory_urls)),
|
||||
re_path(r'^hosts/', include(host_urls)),
|
||||
re_path(r'^host_metrics/', include(host_metric_urls)),
|
||||
# It will be enabled in future version of the AWX
|
||||
# re_path(r'^host_metric_summary_monthly/$', HostMetricSummaryMonthlyList.as_view(), name='host_metric_summary_monthly_list'),
|
||||
re_path(r'^host_metric_summary_monthly/$', HostMetricSummaryMonthlyList.as_view(), name='host_metric_summary_monthly_list'),
|
||||
re_path(r'^groups/', include(group_urls)),
|
||||
re_path(r'^inventory_sources/', include(inventory_source_urls)),
|
||||
re_path(r'^inventory_updates/', include(inventory_update_urls)),
|
||||
|
||||
@ -1564,16 +1564,15 @@ class HostMetricDetail(RetrieveDestroyAPIView):
|
||||
return Response(status=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
# It will be enabled in future version of the AWX
|
||||
# class HostMetricSummaryMonthlyList(ListAPIView):
|
||||
# name = _("Host Metrics Summary Monthly")
|
||||
# model = models.HostMetricSummaryMonthly
|
||||
# serializer_class = serializers.HostMetricSummaryMonthlySerializer
|
||||
# permission_classes = (IsSystemAdminOrAuditor,)
|
||||
# search_fields = ('date',)
|
||||
#
|
||||
# def get_queryset(self):
|
||||
# return self.model.objects.all()
|
||||
class HostMetricSummaryMonthlyList(ListAPIView):
|
||||
name = _("Host Metrics Summary Monthly")
|
||||
model = models.HostMetricSummaryMonthly
|
||||
serializer_class = serializers.HostMetricSummaryMonthlySerializer
|
||||
permission_classes = (IsSystemAdminOrAuditor,)
|
||||
search_fields = ('date',)
|
||||
|
||||
def get_queryset(self):
|
||||
return self.model.objects.all()
|
||||
|
||||
|
||||
class HostList(HostRelatedSearchMixin, ListCreateAPIView):
|
||||
|
||||
@ -107,8 +107,7 @@ class ApiVersionRootView(APIView):
|
||||
data['groups'] = reverse('api:group_list', request=request)
|
||||
data['hosts'] = reverse('api:host_list', request=request)
|
||||
data['host_metrics'] = reverse('api:host_metric_list', request=request)
|
||||
# It will be enabled in future version of the AWX
|
||||
# data['host_metric_summary_monthly'] = reverse('api:host_metric_summary_monthly_list', request=request)
|
||||
data['host_metric_summary_monthly'] = reverse('api:host_metric_summary_monthly_list', request=request)
|
||||
data['job_templates'] = reverse('api:job_template_list', request=request)
|
||||
data['jobs'] = reverse('api:job_list', request=request)
|
||||
data['ad_hoc_commands'] = reverse('api:ad_hoc_command_list', request=request)
|
||||
|
||||
@ -862,6 +862,15 @@ register(
|
||||
category_slug='system',
|
||||
)
|
||||
|
||||
register(
|
||||
'HOST_METRIC_SUMMARY_TASK_LAST_TS',
|
||||
field_class=fields.DateTimeField,
|
||||
label=_('Last computing date of HostMetricSummaryMonthly'),
|
||||
allow_null=True,
|
||||
category=_('System'),
|
||||
category_slug='system',
|
||||
)
|
||||
|
||||
register(
|
||||
'AWX_CLEANUP_PATHS',
|
||||
field_class=fields.BooleanField,
|
||||
|
||||
@ -17,6 +17,6 @@ class Command(BaseCommand):
|
||||
months_ago = options.get('months-ago') or None
|
||||
|
||||
if not months_ago:
|
||||
months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_THRESHOLD', 12)
|
||||
months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_SOFT_THRESHOLD', 12)
|
||||
|
||||
HostMetric.cleanup_task(months_ago)
|
||||
|
||||
@ -0,0 +1,9 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
from awx.main.tasks.host_metrics import HostMetricSummaryMonthlyTask
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Computing of HostMetricSummaryMonthly'
|
||||
|
||||
def handle(self, *args, **options):
|
||||
HostMetricSummaryMonthlyTask().execute()
|
||||
@ -899,18 +899,18 @@ class HostMetric(models.Model):
|
||||
|
||||
last_automation_before = now() - dateutil.relativedelta.relativedelta(months=months_ago)
|
||||
|
||||
logger.info(f'Cleanup [HostMetric]: soft-deleting records last automated before {last_automation_before}')
|
||||
logger.info(f'cleanup_host_metrics: soft-deleting records last automated before {last_automation_before}')
|
||||
HostMetric.active_objects.filter(last_automation__lt=last_automation_before).update(
|
||||
deleted=True, deleted_counter=models.F('deleted_counter') + 1, last_deleted=now()
|
||||
)
|
||||
settings.CLEANUP_HOST_METRICS_LAST_TS = now()
|
||||
except (TypeError, ValueError):
|
||||
logger.error(f"Cleanup [HostMetric]: months_ago({months_ago}) has to be a positive integer value")
|
||||
logger.error(f"cleanup_host_metrics: months_ago({months_ago}) has to be a positive integer value")
|
||||
|
||||
|
||||
class HostMetricSummaryMonthly(models.Model):
|
||||
"""
|
||||
HostMetric summaries computed by scheduled task <TODO> monthly
|
||||
HostMetric summaries computed by scheduled task 'awx.main.tasks.system.host_metric_summary_monthly' monthly
|
||||
"""
|
||||
|
||||
date = models.DateField(unique=True)
|
||||
|
||||
@ -1 +1 @@
|
||||
from . import jobs, receptor, system # noqa
|
||||
from . import host_metrics, jobs, receptor, system # noqa
|
||||
|
||||
205
awx/main/tasks/host_metrics.py
Normal file
205
awx/main/tasks/host_metrics.py
Normal file
@ -0,0 +1,205 @@
|
||||
import datetime
|
||||
from dateutil.relativedelta import relativedelta
|
||||
import logging
|
||||
|
||||
from django.conf import settings
|
||||
from django.db.models import Count
|
||||
from django.db.models.functions import TruncMonth
|
||||
from django.utils.timezone import now
|
||||
from rest_framework.fields import DateTimeField
|
||||
from awx.main.dispatch import get_task_queuename
|
||||
from awx.main.dispatch.publish import task
|
||||
from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly
|
||||
from awx.conf.license import get_license
|
||||
|
||||
logger = logging.getLogger('awx.main.tasks.host_metric_summary_monthly')
|
||||
|
||||
|
||||
@task(queue=get_task_queuename)
|
||||
def host_metric_summary_monthly():
|
||||
"""Run cleanup host metrics summary monthly task each week"""
|
||||
if _is_run_threshold_reached(
|
||||
getattr(settings, 'HOST_METRIC_SUMMARY_TASK_LAST_TS', None), getattr(settings, 'HOST_METRIC_SUMMARY_TASK_INTERVAL', 7) * 86400
|
||||
):
|
||||
logger.info(f"Executing host_metric_summary_monthly, last ran at {getattr(settings, 'HOST_METRIC_SUMMARY_TASK_LAST_TS', '---')}")
|
||||
HostMetricSummaryMonthlyTask().execute()
|
||||
logger.info("Finished host_metric_summary_monthly")
|
||||
|
||||
|
||||
def _is_run_threshold_reached(setting, threshold_seconds):
|
||||
last_time = DateTimeField().to_internal_value(setting) if setting else DateTimeField().to_internal_value('1970-01-01')
|
||||
|
||||
return (now() - last_time).total_seconds() > threshold_seconds
|
||||
|
||||
|
||||
class HostMetricSummaryMonthlyTask:
|
||||
"""
|
||||
This task computes last [threshold] months of HostMetricSummaryMonthly table
|
||||
[threshold] is setting CLEANUP_HOST_METRICS_HARD_THRESHOLD
|
||||
Each record in the table represents changes in HostMetric table in one month
|
||||
It always overrides all the months newer than <threshold>, never updates older months
|
||||
Algorithm:
|
||||
- hosts_added are HostMetric records with first_automation in given month
|
||||
- hosts_deleted are HostMetric records with deleted=True and last_deleted in given month
|
||||
- - HostMetrics soft-deleted before <threshold> also increases hosts_deleted in their last_deleted month
|
||||
- license_consumed is license_consumed(previous month) + hosts_added - hosts_deleted
|
||||
- - license_consumed for HostMetricSummaryMonthly.date < [threshold] is computed also from
|
||||
all HostMetrics.first_automation < [threshold]
|
||||
- license_capacity is set only for current month, and it's never updated (value taken from current subscription)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.host_metrics = {}
|
||||
self.processed_month = self._get_first_month()
|
||||
self.existing_summaries = None
|
||||
self.existing_summaries_idx = 0
|
||||
self.existing_summaries_cnt = 0
|
||||
self.records_to_create = []
|
||||
self.records_to_update = []
|
||||
|
||||
def execute(self):
|
||||
self._load_existing_summaries()
|
||||
self._load_hosts_added()
|
||||
self._load_hosts_deleted()
|
||||
|
||||
# Get first month after last hard delete
|
||||
month = self._get_first_month()
|
||||
license_consumed = self._get_license_consumed_before(month)
|
||||
|
||||
# Fill record for each month
|
||||
while month <= datetime.date.today().replace(day=1):
|
||||
summary = self._find_or_create_summary(month)
|
||||
# Update summary and update license_consumed by hosts added/removed this month
|
||||
self._update_summary(summary, month, license_consumed)
|
||||
license_consumed = summary.license_consumed
|
||||
|
||||
month = month + relativedelta(months=1)
|
||||
|
||||
# Create/Update stats
|
||||
HostMetricSummaryMonthly.objects.bulk_create(self.records_to_create, batch_size=1000)
|
||||
HostMetricSummaryMonthly.objects.bulk_update(self.records_to_update, ['license_consumed', 'hosts_added', 'hosts_deleted'], batch_size=1000)
|
||||
|
||||
# Set timestamp of last run
|
||||
settings.HOST_METRIC_SUMMARY_TASK_LAST_TS = now()
|
||||
|
||||
def _get_license_consumed_before(self, month):
|
||||
license_consumed = 0
|
||||
for metric_month, metric in self.host_metrics.items():
|
||||
if metric_month < month:
|
||||
hosts_added = metric.get('hosts_added', 0)
|
||||
hosts_deleted = metric.get('hosts_deleted', 0)
|
||||
license_consumed = license_consumed + hosts_added - hosts_deleted
|
||||
else:
|
||||
break
|
||||
return license_consumed
|
||||
|
||||
def _load_existing_summaries(self):
|
||||
"""Find all summaries newer than host metrics delete threshold"""
|
||||
self.existing_summaries = HostMetricSummaryMonthly.objects.filter(date__gte=self._get_first_month()).order_by('date')
|
||||
self.existing_summaries_idx = 0
|
||||
self.existing_summaries_cnt = len(self.existing_summaries)
|
||||
|
||||
def _load_hosts_added(self):
|
||||
"""Aggregates hosts added each month, by the 'first_automation' timestamp"""
|
||||
#
|
||||
# -- SQL translation (for better code readability)
|
||||
# SELECT date_trunc('month', first_automation) as month,
|
||||
# count(first_automation) AS hosts_added
|
||||
# FROM main_hostmetric
|
||||
# GROUP BY month
|
||||
# ORDER by month;
|
||||
result = (
|
||||
HostMetric.objects.annotate(month=TruncMonth('first_automation'))
|
||||
.values('month')
|
||||
.annotate(hosts_added=Count('first_automation'))
|
||||
.values('month', 'hosts_added')
|
||||
.order_by('month')
|
||||
)
|
||||
|
||||
for host_metric in list(result):
|
||||
month = host_metric['month']
|
||||
if month:
|
||||
beginning_of_month = datetime.date(month.year, month.month, 1)
|
||||
if self.host_metrics.get(beginning_of_month) is None:
|
||||
self.host_metrics[beginning_of_month] = {}
|
||||
self.host_metrics[beginning_of_month]['hosts_added'] = host_metric['hosts_added']
|
||||
|
||||
def _load_hosts_deleted(self):
|
||||
"""
|
||||
Aggregates hosts deleted each month, by the 'last_deleted' timestamp.
|
||||
Host metrics have to be deleted NOW to be counted as deleted before
|
||||
(by intention - statistics can change retrospectively by re-automation of previously deleted host)
|
||||
"""
|
||||
#
|
||||
# -- SQL translation (for better code readability)
|
||||
# SELECT date_trunc('month', last_deleted) as month,
|
||||
# count(last_deleted) AS hosts_deleted
|
||||
# FROM main_hostmetric
|
||||
# WHERE deleted = True
|
||||
# GROUP BY 1 # equal to "GROUP BY month"
|
||||
# ORDER by month;
|
||||
result = (
|
||||
HostMetric.objects.annotate(month=TruncMonth('last_deleted'))
|
||||
.values('month')
|
||||
.annotate(hosts_deleted=Count('last_deleted'))
|
||||
.values('month', 'hosts_deleted')
|
||||
.filter(deleted=True)
|
||||
.order_by('month')
|
||||
)
|
||||
for host_metric in list(result):
|
||||
month = host_metric['month']
|
||||
if month:
|
||||
beginning_of_month = datetime.date(month.year, month.month, 1)
|
||||
if self.host_metrics.get(beginning_of_month) is None:
|
||||
self.host_metrics[beginning_of_month] = {}
|
||||
self.host_metrics[beginning_of_month]['hosts_deleted'] = host_metric['hosts_deleted']
|
||||
|
||||
def _find_or_create_summary(self, month):
|
||||
summary = self._find_summary(month)
|
||||
|
||||
if not summary:
|
||||
summary = HostMetricSummaryMonthly(date=month)
|
||||
self.records_to_create.append(summary)
|
||||
else:
|
||||
self.records_to_update.append(summary)
|
||||
return summary
|
||||
|
||||
def _find_summary(self, month):
|
||||
"""
|
||||
Existing summaries are ordered by month ASC.
|
||||
This method is called with month in ascending order too => only 1 traversing is enough
|
||||
"""
|
||||
summary = None
|
||||
while not summary and self.existing_summaries_idx < self.existing_summaries_cnt:
|
||||
tmp = self.existing_summaries[self.existing_summaries_idx]
|
||||
if tmp.date < month:
|
||||
self.existing_summaries_idx += 1
|
||||
elif tmp.date == month:
|
||||
summary = tmp
|
||||
elif tmp.date > month:
|
||||
break
|
||||
return summary
|
||||
|
||||
def _update_summary(self, summary, month, license_consumed):
|
||||
"""Updates the metric with hosts added and deleted and set license info for current month"""
|
||||
# Get month counts from host metrics, zero if not found
|
||||
hosts_added, hosts_deleted = 0, 0
|
||||
if metric := self.host_metrics.get(month, None):
|
||||
hosts_added = metric.get('hosts_added', 0)
|
||||
hosts_deleted = metric.get('hosts_deleted', 0)
|
||||
|
||||
summary.license_consumed = license_consumed + hosts_added - hosts_deleted
|
||||
summary.hosts_added = hosts_added
|
||||
summary.hosts_deleted = hosts_deleted
|
||||
|
||||
# Set subscription count for current month
|
||||
if month == datetime.date.today().replace(day=1):
|
||||
license_info = get_license()
|
||||
summary.license_capacity = license_info.get('instance_count', 0)
|
||||
return summary
|
||||
|
||||
@staticmethod
|
||||
def _get_first_month():
|
||||
"""Returns first month after host metrics hard delete threshold"""
|
||||
threshold = getattr(settings, 'CLEANUP_HOST_METRICS_HARD_THRESHOLD', 36)
|
||||
return datetime.date.today().replace(day=1) - relativedelta(months=int(threshold) - 1)
|
||||
@ -316,13 +316,8 @@ def send_notifications(notification_list, job_id=None):
|
||||
@task(queue=get_task_queuename)
|
||||
def gather_analytics():
|
||||
from awx.conf.models import Setting
|
||||
from rest_framework.fields import DateTimeField
|
||||
|
||||
last_gather = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_GATHER').first()
|
||||
last_time = DateTimeField().to_internal_value(last_gather.value) if last_gather and last_gather.value else None
|
||||
gather_time = now()
|
||||
|
||||
if not last_time or ((gather_time - last_time).total_seconds() > settings.AUTOMATION_ANALYTICS_GATHER_INTERVAL):
|
||||
if is_run_threshold_reached(Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_GATHER').first(), settings.AUTOMATION_ANALYTICS_GATHER_INTERVAL):
|
||||
analytics.gather()
|
||||
|
||||
|
||||
@ -381,16 +376,25 @@ def cleanup_images_and_files():
|
||||
|
||||
@task(queue=get_task_queuename)
|
||||
def cleanup_host_metrics():
|
||||
"""Run cleanup host metrics ~each month"""
|
||||
# TODO: move whole method to host_metrics in follow-up PR
|
||||
from awx.conf.models import Setting
|
||||
|
||||
if is_run_threshold_reached(
|
||||
Setting.objects.filter(key='CLEANUP_HOST_METRICS_LAST_TS').first(), getattr(settings, 'CLEANUP_HOST_METRICS_INTERVAL', 30) * 86400
|
||||
):
|
||||
months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_SOFT_THRESHOLD', 12)
|
||||
logger.info("Executing cleanup_host_metrics")
|
||||
HostMetric.cleanup_task(months_ago)
|
||||
logger.info("Finished cleanup_host_metrics")
|
||||
|
||||
|
||||
def is_run_threshold_reached(setting, threshold_seconds):
|
||||
from rest_framework.fields import DateTimeField
|
||||
|
||||
last_cleanup = Setting.objects.filter(key='CLEANUP_HOST_METRICS_LAST_TS').first()
|
||||
last_time = DateTimeField().to_internal_value(last_cleanup.value) if last_cleanup and last_cleanup.value else None
|
||||
last_time = DateTimeField().to_internal_value(setting.value) if setting and setting.value else DateTimeField().to_internal_value('1970-01-01')
|
||||
|
||||
cleanup_interval_secs = getattr(settings, 'CLEANUP_HOST_METRICS_INTERVAL', 30) * 86400
|
||||
if not last_time or ((now() - last_time).total_seconds() > cleanup_interval_secs):
|
||||
months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_THRESHOLD', 12)
|
||||
HostMetric.cleanup_task(months_ago)
|
||||
return (now() - last_time).total_seconds() > threshold_seconds
|
||||
|
||||
|
||||
@task(queue=get_task_queuename)
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
import json
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from django.core.exceptions import ValidationError
|
||||
|
||||
from unittest import mock
|
||||
|
||||
from awx.main.models import (
|
||||
Organization,
|
||||
@ -20,6 +23,7 @@ from awx.main.models import (
|
||||
WorkflowJobNode,
|
||||
WorkflowJobTemplateNode,
|
||||
)
|
||||
from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly
|
||||
|
||||
# mk methods should create only a single object of a single type.
|
||||
# they should also have the option of being persisted or not.
|
||||
@ -248,3 +252,42 @@ def mk_workflow_job_node(unified_job_template=None, success_nodes=None, failure_
|
||||
if persisted:
|
||||
workflow_node.save()
|
||||
return workflow_node
|
||||
|
||||
|
||||
def mk_host_metric(hostname, first_automation, last_automation=None, last_deleted=None, deleted=False, persisted=True):
|
||||
ok, idx = False, 1
|
||||
while not ok:
|
||||
try:
|
||||
with mock.patch("django.utils.timezone.now") as mock_now:
|
||||
mock_now.return_value = first_automation
|
||||
metric = HostMetric(
|
||||
hostname=hostname or f"host-{first_automation}-{idx}",
|
||||
first_automation=first_automation,
|
||||
last_automation=last_automation or first_automation,
|
||||
last_deleted=last_deleted,
|
||||
deleted=deleted,
|
||||
)
|
||||
metric.validate_unique()
|
||||
if persisted:
|
||||
metric.save()
|
||||
ok = True
|
||||
except ValidationError as e:
|
||||
# Repeat create for auto-generated hostname
|
||||
if not hostname and e.message_dict.get('hostname', None):
|
||||
idx += 1
|
||||
else:
|
||||
raise e
|
||||
|
||||
|
||||
def mk_host_metric_summary(date, license_consumed=0, license_capacity=0, hosts_added=0, hosts_deleted=0, indirectly_managed_hosts=0, persisted=True):
|
||||
summary = HostMetricSummaryMonthly(
|
||||
date=date,
|
||||
license_consumed=license_consumed,
|
||||
license_capacity=license_capacity,
|
||||
hosts_added=hosts_added,
|
||||
hosts_deleted=hosts_deleted,
|
||||
indirectly_managed_hosts=indirectly_managed_hosts,
|
||||
)
|
||||
if persisted:
|
||||
summary.save()
|
||||
return summary
|
||||
|
||||
@ -0,0 +1,382 @@
|
||||
import pytest
|
||||
import datetime
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
|
||||
|
||||
from awx.main.management.commands.host_metric_summary_monthly import Command
|
||||
from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly
|
||||
from awx.main.tests.factories.fixtures import mk_host_metric, mk_host_metric_summary
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def threshold():
|
||||
return int(getattr(settings, 'CLEANUP_HOST_METRICS_HARD_THRESHOLD', 36))
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize("metrics_cnt", [0, 1, 2, 3])
|
||||
@pytest.mark.parametrize("mode", ["old_data", "actual_data", "all_data"])
|
||||
def test_summaries_counts(threshold, metrics_cnt, mode):
|
||||
assert HostMetricSummaryMonthly.objects.count() == 0
|
||||
|
||||
for idx in range(metrics_cnt):
|
||||
if mode == "old_data" or mode == "all_data":
|
||||
mk_host_metric(None, months_ago(threshold + idx, "dt"))
|
||||
elif mode == "actual_data" or mode == "all_data":
|
||||
mk_host_metric(None, (months_ago(threshold - idx, "dt")))
|
||||
|
||||
Command().handle()
|
||||
|
||||
# Number of records is equal to host metrics' hard cleanup months
|
||||
assert HostMetricSummaryMonthly.objects.count() == threshold
|
||||
|
||||
# Records start with date in the month following to the threshold month
|
||||
date = months_ago(threshold - 1)
|
||||
for metric in list(HostMetricSummaryMonthly.objects.order_by('date').all()):
|
||||
assert metric.date == date
|
||||
date += relativedelta(months=1)
|
||||
|
||||
# Older record are untouched
|
||||
mk_host_metric_summary(date=months_ago(threshold + 10))
|
||||
Command().handle()
|
||||
|
||||
assert HostMetricSummaryMonthly.objects.count() == threshold + 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize("mode", ["old_data", "actual_data", "all_data"])
|
||||
def test_summary_values(threshold, mode):
|
||||
tester = {"old_data": MetricsTesterOldData(threshold), "actual_data": MetricsTesterActualData(threshold), "all_data": MetricsTesterCombinedData(threshold)}[
|
||||
mode
|
||||
]
|
||||
|
||||
for iteration in ["create_metrics", "add_old_summaries", "change_metrics", "delete_metrics", "add_metrics"]:
|
||||
getattr(tester, iteration)() # call method by string
|
||||
|
||||
# Operation is idempotent, repeat twice
|
||||
for _ in range(2):
|
||||
Command().handle()
|
||||
# call assert method by string
|
||||
getattr(tester, f"assert_{iteration}")()
|
||||
|
||||
|
||||
class MetricsTester:
|
||||
def __init__(self, threshold, ignore_asserts=False):
|
||||
self.threshold = threshold
|
||||
self.expected_summaries = {}
|
||||
self.ignore_asserts = ignore_asserts
|
||||
|
||||
def add_old_summaries(self):
|
||||
"""These records don't correspond with Host metrics"""
|
||||
mk_host_metric_summary(self.below(4), license_consumed=100, hosts_added=10, hosts_deleted=5)
|
||||
mk_host_metric_summary(self.below(3), license_consumed=105, hosts_added=20, hosts_deleted=10)
|
||||
mk_host_metric_summary(self.below(2), license_consumed=115, hosts_added=60, hosts_deleted=75)
|
||||
|
||||
def assert_add_old_summaries(self):
|
||||
"""Old summary records should be untouched"""
|
||||
self.expected_summaries[self.below(4)] = {"date": self.below(4), "license_consumed": 100, "hosts_added": 10, "hosts_deleted": 5}
|
||||
self.expected_summaries[self.below(3)] = {"date": self.below(3), "license_consumed": 105, "hosts_added": 20, "hosts_deleted": 10}
|
||||
self.expected_summaries[self.below(2)] = {"date": self.below(2), "license_consumed": 115, "hosts_added": 60, "hosts_deleted": 75}
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
def assert_host_metric_summaries(self):
|
||||
"""Ignore asserts when old/actual test object is used only as a helper for Combined test"""
|
||||
if self.ignore_asserts:
|
||||
return True
|
||||
|
||||
for summary in list(HostMetricSummaryMonthly.objects.order_by('date').all()):
|
||||
assert self.expected_summaries.get(summary.date, None) is not None
|
||||
|
||||
assert self.expected_summaries[summary.date] == {
|
||||
"date": summary.date,
|
||||
"license_consumed": summary.license_consumed,
|
||||
"hosts_added": summary.hosts_added,
|
||||
"hosts_deleted": summary.hosts_deleted,
|
||||
}
|
||||
|
||||
def below(self, months, fmt="date"):
|
||||
"""months below threshold, returns first date of that month"""
|
||||
date = months_ago(self.threshold + months)
|
||||
if fmt == "dt":
|
||||
return timezone.make_aware(datetime.datetime.combine(date, datetime.datetime.min.time()))
|
||||
else:
|
||||
return date
|
||||
|
||||
def above(self, months, fmt="date"):
|
||||
"""months above threshold, returns first date of that month"""
|
||||
date = months_ago(self.threshold - months)
|
||||
if fmt == "dt":
|
||||
return timezone.make_aware(datetime.datetime.combine(date, datetime.datetime.min.time()))
|
||||
else:
|
||||
return date
|
||||
|
||||
|
||||
class MetricsTesterOldData(MetricsTester):
|
||||
def create_metrics(self):
|
||||
"""Creates 7 host metrics older than delete threshold"""
|
||||
mk_host_metric("host_1", first_automation=self.below(3, "dt"))
|
||||
mk_host_metric("host_2", first_automation=self.below(2, "dt"))
|
||||
mk_host_metric("host_3", first_automation=self.below(2, "dt"), last_deleted=self.above(2, "dt"), deleted=False)
|
||||
mk_host_metric("host_4", first_automation=self.below(2, "dt"), last_deleted=self.above(2, "dt"), deleted=True)
|
||||
mk_host_metric("host_5", first_automation=self.below(2, "dt"), last_deleted=self.below(2, "dt"), deleted=True)
|
||||
mk_host_metric("host_6", first_automation=self.below(1, "dt"), last_deleted=self.below(1, "dt"), deleted=False)
|
||||
mk_host_metric("host_7", first_automation=self.below(1, "dt"))
|
||||
|
||||
def assert_create_metrics(self):
|
||||
"""
|
||||
Month 1 is computed from older host metrics,
|
||||
Month 2 has deletion (host_4)
|
||||
Other months are unchanged (same as month 2)
|
||||
"""
|
||||
self.expected_summaries = {
|
||||
self.above(1): {"date": self.above(1), "license_consumed": 6, "hosts_added": 0, "hosts_deleted": 0},
|
||||
self.above(2): {"date": self.above(2), "license_consumed": 5, "hosts_added": 0, "hosts_deleted": 1},
|
||||
}
|
||||
# no change in months 3+
|
||||
idx = 3
|
||||
month = self.above(idx)
|
||||
while month <= beginning_of_the_month():
|
||||
self.expected_summaries[self.above(idx)] = {"date": self.above(idx), "license_consumed": 5, "hosts_added": 0, "hosts_deleted": 0}
|
||||
month += relativedelta(months=1)
|
||||
idx += 1
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
def add_old_summaries(self):
|
||||
super().add_old_summaries()
|
||||
|
||||
def assert_add_old_summaries(self):
|
||||
super().assert_add_old_summaries()
|
||||
|
||||
@staticmethod
|
||||
def change_metrics():
|
||||
"""Hosts 1,2 soft deleted, host_4 automated again (undeleted)"""
|
||||
HostMetric.objects.filter(hostname='host_1').update(last_deleted=beginning_of_the_month("dt"), deleted=True)
|
||||
HostMetric.objects.filter(hostname='host_2').update(last_deleted=timezone.now(), deleted=True)
|
||||
HostMetric.objects.filter(hostname='host_4').update(deleted=False)
|
||||
|
||||
def assert_change_metrics(self):
|
||||
"""
|
||||
Summaries since month 2 were changed (host_4 restored == automated again)
|
||||
Current month has 2 deletions (host_1, host_2)
|
||||
"""
|
||||
self.expected_summaries[self.above(2)] |= {'hosts_deleted': 0}
|
||||
for idx in range(2, self.threshold):
|
||||
self.expected_summaries[self.above(idx)] |= {'license_consumed': 6}
|
||||
self.expected_summaries[beginning_of_the_month()] |= {'license_consumed': 4, 'hosts_deleted': 2}
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
@staticmethod
|
||||
def delete_metrics():
|
||||
"""Deletes metric deleted before the threshold"""
|
||||
HostMetric.objects.filter(hostname='host_5').delete()
|
||||
|
||||
def assert_delete_metrics(self):
|
||||
"""No change"""
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
@staticmethod
|
||||
def add_metrics():
|
||||
"""Adds new metrics"""
|
||||
mk_host_metric("host_24", first_automation=beginning_of_the_month("dt"))
|
||||
mk_host_metric("host_25", first_automation=beginning_of_the_month("dt")) # timezone.now())
|
||||
|
||||
def assert_add_metrics(self):
|
||||
"""Summary in current month is updated"""
|
||||
self.expected_summaries[beginning_of_the_month()]['license_consumed'] = 6
|
||||
self.expected_summaries[beginning_of_the_month()]['hosts_added'] = 2
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
|
||||
class MetricsTesterActualData(MetricsTester):
|
||||
def create_metrics(self):
|
||||
"""Creates 16 host metrics newer than delete threshold"""
|
||||
mk_host_metric("host_8", first_automation=self.above(1, "dt"))
|
||||
mk_host_metric("host_9", first_automation=self.above(1, "dt"), last_deleted=self.above(1, "dt"))
|
||||
mk_host_metric("host_10", first_automation=self.above(1, "dt"), last_deleted=self.above(1, "dt"), deleted=True)
|
||||
mk_host_metric("host_11", first_automation=self.above(1, "dt"), last_deleted=self.above(2, "dt"))
|
||||
mk_host_metric("host_12", first_automation=self.above(1, "dt"), last_deleted=self.above(2, "dt"), deleted=True)
|
||||
mk_host_metric("host_13", first_automation=self.above(2, "dt"))
|
||||
mk_host_metric("host_14", first_automation=self.above(2, "dt"), last_deleted=self.above(2, "dt"))
|
||||
mk_host_metric("host_15", first_automation=self.above(2, "dt"), last_deleted=self.above(2, "dt"), deleted=True)
|
||||
mk_host_metric("host_16", first_automation=self.above(2, "dt"), last_deleted=self.above(3, "dt"))
|
||||
mk_host_metric("host_17", first_automation=self.above(2, "dt"), last_deleted=self.above(3, "dt"), deleted=True)
|
||||
mk_host_metric("host_18", first_automation=self.above(4, "dt"))
|
||||
# next one shouldn't happen in real (deleted=True, last_deleted = NULL)
|
||||
mk_host_metric("host_19", first_automation=self.above(4, "dt"), deleted=True)
|
||||
mk_host_metric("host_20", first_automation=self.above(4, "dt"), last_deleted=self.above(4, "dt"))
|
||||
mk_host_metric("host_21", first_automation=self.above(4, "dt"), last_deleted=self.above(4, "dt"), deleted=True)
|
||||
mk_host_metric("host_22", first_automation=self.above(4, "dt"), last_deleted=self.above(5, "dt"))
|
||||
mk_host_metric("host_23", first_automation=self.above(4, "dt"), last_deleted=self.above(5, "dt"), deleted=True)
|
||||
|
||||
def assert_create_metrics(self):
|
||||
self.expected_summaries = {
|
||||
self.above(1): {"date": self.above(1), "license_consumed": 4, "hosts_added": 5, "hosts_deleted": 1},
|
||||
self.above(2): {"date": self.above(2), "license_consumed": 7, "hosts_added": 5, "hosts_deleted": 2},
|
||||
self.above(3): {"date": self.above(3), "license_consumed": 6, "hosts_added": 0, "hosts_deleted": 1},
|
||||
self.above(4): {"date": self.above(4), "license_consumed": 11, "hosts_added": 6, "hosts_deleted": 1},
|
||||
self.above(5): {"date": self.above(5), "license_consumed": 10, "hosts_added": 0, "hosts_deleted": 1},
|
||||
}
|
||||
# no change in months 6+
|
||||
idx = 6
|
||||
month = self.above(idx)
|
||||
while month <= beginning_of_the_month():
|
||||
self.expected_summaries[self.above(idx)] = {"date": self.above(idx), "license_consumed": 10, "hosts_added": 0, "hosts_deleted": 0}
|
||||
month += relativedelta(months=1)
|
||||
idx += 1
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
def add_old_summaries(self):
|
||||
super().add_old_summaries()
|
||||
|
||||
def assert_add_old_summaries(self):
|
||||
super().assert_add_old_summaries()
|
||||
|
||||
@staticmethod
|
||||
def change_metrics():
|
||||
"""
|
||||
- Hosts 12, 19, 21 were automated again (undeleted)
|
||||
- Host 16 was soft deleted
|
||||
- Host 17 was undeleted and soft deleted again
|
||||
"""
|
||||
HostMetric.objects.filter(hostname='host_12').update(deleted=False)
|
||||
HostMetric.objects.filter(hostname='host_16').update(last_deleted=timezone.now(), deleted=True)
|
||||
HostMetric.objects.filter(hostname='host_17').update(last_deleted=beginning_of_the_month("dt"), deleted=True)
|
||||
HostMetric.objects.filter(hostname='host_19').update(deleted=False)
|
||||
HostMetric.objects.filter(hostname='host_21').update(deleted=False)
|
||||
|
||||
def assert_change_metrics(self):
|
||||
"""
|
||||
Summaries since month 2 were changed
|
||||
Current month has 2 deletions (host_16, host_17)
|
||||
"""
|
||||
self.expected_summaries[self.above(2)] |= {'license_consumed': 8, 'hosts_deleted': 1}
|
||||
self.expected_summaries[self.above(3)] |= {'license_consumed': 8, 'hosts_deleted': 0}
|
||||
self.expected_summaries[self.above(4)] |= {'license_consumed': 14, 'hosts_deleted': 0}
|
||||
|
||||
# month 5 had hosts_deleted 1 => license_consumed == 14 - 1
|
||||
for idx in range(5, self.threshold):
|
||||
self.expected_summaries[self.above(idx)] |= {'license_consumed': 13}
|
||||
self.expected_summaries[beginning_of_the_month()] |= {'license_consumed': 11, 'hosts_deleted': 2}
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
def delete_metrics(self):
|
||||
"""Hard cleanup can't delete metrics newer than threshold. No change"""
|
||||
pass
|
||||
|
||||
def assert_delete_metrics(self):
|
||||
"""No change"""
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
@staticmethod
|
||||
def add_metrics():
|
||||
"""Adds new metrics"""
|
||||
mk_host_metric("host_26", first_automation=beginning_of_the_month("dt"))
|
||||
mk_host_metric("host_27", first_automation=timezone.now())
|
||||
|
||||
def assert_add_metrics(self):
|
||||
"""
|
||||
Two metrics were deleted in current month by change_metrics()
|
||||
Two metrics are added now
|
||||
=> license_consumed is equal to the previous month (13 - 2 + 2)
|
||||
"""
|
||||
self.expected_summaries[beginning_of_the_month()] |= {'license_consumed': 13, 'hosts_added': 2}
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
|
||||
class MetricsTesterCombinedData(MetricsTester):
|
||||
def __init__(self, threshold):
|
||||
super().__init__(threshold)
|
||||
self.old_data = MetricsTesterOldData(threshold, ignore_asserts=True)
|
||||
self.actual_data = MetricsTesterActualData(threshold, ignore_asserts=True)
|
||||
|
||||
def assert_host_metric_summaries(self):
|
||||
self._combine_expected_summaries()
|
||||
super().assert_host_metric_summaries()
|
||||
|
||||
def create_metrics(self):
|
||||
self.old_data.create_metrics()
|
||||
self.actual_data.create_metrics()
|
||||
|
||||
def assert_create_metrics(self):
|
||||
self.old_data.assert_create_metrics()
|
||||
self.actual_data.assert_create_metrics()
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
def add_old_summaries(self):
|
||||
super().add_old_summaries()
|
||||
|
||||
def assert_add_old_summaries(self):
|
||||
self.old_data.assert_add_old_summaries()
|
||||
self.actual_data.assert_add_old_summaries()
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
def change_metrics(self):
|
||||
self.old_data.change_metrics()
|
||||
self.actual_data.change_metrics()
|
||||
|
||||
def assert_change_metrics(self):
|
||||
self.old_data.assert_change_metrics()
|
||||
self.actual_data.assert_change_metrics()
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
def delete_metrics(self):
|
||||
self.old_data.delete_metrics()
|
||||
self.actual_data.delete_metrics()
|
||||
|
||||
def assert_delete_metrics(self):
|
||||
self.old_data.assert_delete_metrics()
|
||||
self.actual_data.assert_delete_metrics()
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
def add_metrics(self):
|
||||
self.old_data.add_metrics()
|
||||
self.actual_data.add_metrics()
|
||||
|
||||
def assert_add_metrics(self):
|
||||
self.old_data.assert_add_metrics()
|
||||
self.actual_data.assert_add_metrics()
|
||||
|
||||
self.assert_host_metric_summaries()
|
||||
|
||||
def _combine_expected_summaries(self):
|
||||
"""
|
||||
Expected summaries are sum of expected values for tests with old and actual data
|
||||
Except data older than hard delete threshold (these summaries are untouched by task => the same in all tests)
|
||||
"""
|
||||
for date, summary in self.old_data.expected_summaries.items():
|
||||
if date <= months_ago(self.threshold):
|
||||
license_consumed = summary['license_consumed']
|
||||
hosts_added = summary['hosts_added']
|
||||
hosts_deleted = summary['hosts_deleted']
|
||||
else:
|
||||
license_consumed = summary['license_consumed'] + self.actual_data.expected_summaries[date]['license_consumed']
|
||||
hosts_added = summary['hosts_added'] + self.actual_data.expected_summaries[date]['hosts_added']
|
||||
hosts_deleted = summary['hosts_deleted'] + self.actual_data.expected_summaries[date]['hosts_deleted']
|
||||
self.expected_summaries[date] = {'date': date, 'license_consumed': license_consumed, 'hosts_added': hosts_added, 'hosts_deleted': hosts_deleted}
|
||||
|
||||
|
||||
def months_ago(num, fmt="date"):
|
||||
if num is None:
|
||||
return None
|
||||
return beginning_of_the_month(fmt) - relativedelta(months=num)
|
||||
|
||||
|
||||
def beginning_of_the_month(fmt="date"):
|
||||
date = datetime.date.today().replace(day=1)
|
||||
if fmt == "dt":
|
||||
return timezone.make_aware(datetime.datetime.combine(date, datetime.datetime.min.time()))
|
||||
else:
|
||||
return date
|
||||
@ -471,7 +471,8 @@ CELERYBEAT_SCHEDULE = {
|
||||
'receptor_reaper': {'task': 'awx.main.tasks.system.awx_receptor_workunit_reaper', 'schedule': timedelta(seconds=60)},
|
||||
'send_subsystem_metrics': {'task': 'awx.main.analytics.analytics_tasks.send_subsystem_metrics', 'schedule': timedelta(seconds=20)},
|
||||
'cleanup_images': {'task': 'awx.main.tasks.system.cleanup_images_and_files', 'schedule': timedelta(hours=3)},
|
||||
'cleanup_host_metrics': {'task': 'awx.main.tasks.system.cleanup_host_metrics', 'schedule': timedelta(days=1)},
|
||||
'cleanup_host_metrics': {'task': 'awx.main.tasks.system.cleanup_host_metrics', 'schedule': timedelta(hours=3, minutes=30)},
|
||||
'host_metric_summary_monthly': {'task': 'awx.main.tasks.host_metrics.host_metric_summary_monthly', 'schedule': timedelta(hours=4)},
|
||||
}
|
||||
|
||||
# Django Caching Configuration
|
||||
@ -1054,4 +1055,12 @@ CLEANUP_HOST_METRICS_LAST_TS = None
|
||||
# Host metrics cleanup - minimal interval between two cleanups in days
|
||||
CLEANUP_HOST_METRICS_INTERVAL = 30 # days
|
||||
# Host metrics cleanup - soft-delete HostMetric records with last_automation < [threshold] (in months)
|
||||
CLEANUP_HOST_METRICS_THRESHOLD = 12 # months
|
||||
CLEANUP_HOST_METRICS_SOFT_THRESHOLD = 12 # months
|
||||
# Host metrics cleanup
|
||||
# - delete HostMetric record with deleted=True and last_deleted < [threshold]
|
||||
# - also threshold for computing HostMetricSummaryMonthly (command/scheduled task)
|
||||
CLEANUP_HOST_METRICS_HARD_THRESHOLD = 36 # months
|
||||
|
||||
# Host metric summary monthly task - last time of run
|
||||
HOST_METRIC_SUMMARY_TASK_LAST_TS = None
|
||||
HOST_METRIC_SUMMARY_TASK_INTERVAL = 7 # days
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user