diff --git a/awx/main/conf.py b/awx/main/conf.py index 33b3b4714d..c9515e845e 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -822,6 +822,15 @@ register( category_slug='system', ) +register( + 'CLEANUP_HOST_METRICS_LAST_TS', + field_class=fields.DateTimeField, + label=_('Last cleanup date for HostMetrics'), + allow_null=True, + category=_('System'), + category_slug='system', +) + def logging_validate(serializer, attrs): if not serializer.instance or not hasattr(serializer.instance, 'LOG_AGGREGATOR_HOST') or not hasattr(serializer.instance, 'LOG_AGGREGATOR_TYPE'): diff --git a/awx/main/management/commands/cleanup_host_metrics.py b/awx/main/management/commands/cleanup_host_metrics.py new file mode 100644 index 0000000000..788ffcba94 --- /dev/null +++ b/awx/main/management/commands/cleanup_host_metrics.py @@ -0,0 +1,22 @@ +from awx.main.models import HostMetric +from django.core.management.base import BaseCommand +from django.conf import settings + + +class Command(BaseCommand): + """ + Run soft-deleting of HostMetrics + """ + + help = 'Run soft-deleting of HostMetrics' + + def add_arguments(self, parser): + parser.add_argument('--months-ago', type=int, dest='months-ago', action='store', help='Threshold in months for soft-deleting') + + def handle(self, *args, **options): + months_ago = options.get('months-ago') or None + + if not months_ago: + months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_THRESHOLD', 12) + + HostMetric.cleanup_task(months_ago) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index fe1b09e23d..626775a9d6 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -9,6 +9,8 @@ import re import copy import os.path from urllib.parse import urljoin + +import dateutil.relativedelta import yaml # Django @@ -888,6 +890,16 @@ class HostMetric(models.Model): self.deleted = False self.save(update_fields=['deleted']) + @classmethod + def cleanup_task(cls, months_ago): + last_automation_before = now() - dateutil.relativedelta.relativedelta(months=months_ago) + + logger.info(f'Cleanup [HostMetric]: soft-deleting records last automated before {last_automation_before}') + HostMetric.objects.filter(last_automation__lt=last_automation_before).update( + deleted=True, deleted_counter=models.F('deleted_counter') + 1, last_deleted=now() + ) + settings.CLEANUP_HOST_METRICS_LAST_TS = now() + class HostMetricSummaryMonthly(models.Model): """ diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index a83dad5ccc..36fc266803 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -47,6 +47,7 @@ from awx.main.models import ( Inventory, SmartInventoryMembership, Job, + HostMetric, ) from awx.main.constants import ACTIVE_STATES from awx.main.dispatch.publish import task @@ -378,6 +379,20 @@ def cleanup_images_and_files(): _cleanup_images_and_files() +@task(queue=get_task_queuename) +def cleanup_host_metrics(): + from awx.conf.models import Setting + from rest_framework.fields import DateTimeField + + last_cleanup = Setting.objects.filter(key='CLEANUP_HOST_METRICS_LAST_TS').first() + last_time = DateTimeField().to_internal_value(last_cleanup.value) if last_cleanup and last_cleanup.value else None + + cleanup_interval_secs = getattr(settings, 'CLEANUP_HOST_METRICS_INTERVAL', 30) * 86400 + if not last_time or ((now() - last_time).total_seconds() > cleanup_interval_secs): + months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_THRESHOLD', 12) + HostMetric.cleanup_task(months_ago) + + @task(queue=get_task_queuename) def cluster_node_health_check(node): """ diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 593e6ae002..82d0e7b343 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -475,6 +475,7 @@ CELERYBEAT_SCHEDULE = { 'receptor_reaper': {'task': 'awx.main.tasks.system.awx_receptor_workunit_reaper', 'schedule': timedelta(seconds=60)}, 'send_subsystem_metrics': {'task': 'awx.main.analytics.analytics_tasks.send_subsystem_metrics', 'schedule': timedelta(seconds=20)}, 'cleanup_images': {'task': 'awx.main.tasks.system.cleanup_images_and_files', 'schedule': timedelta(hours=3)}, + 'cleanup_host_metrics': {'task': 'awx.main.tasks.system.cleanup_host_metrics', 'schedule': timedelta(days=1)}, } # Django Caching Configuration @@ -1052,3 +1053,10 @@ UI_NEXT = True # - '': No model - Subscription not counted from Host Metrics # - 'unique_managed_hosts': Compliant = automated - deleted hosts (using /api/v2/host_metrics/) SUBSCRIPTION_USAGE_MODEL = '' + +# Host metrics cleanup - last time of the cleanup run (soft-deleting records) +CLEANUP_HOST_METRICS_LAST_TS = None +# Host metrics cleanup - minimal interval between two cleanups in days +CLEANUP_HOST_METRICS_INTERVAL = 30 # days +# Host metrics cleanup - soft-delete HostMetric records with last_automation < [threshold] (in months) +CLEANUP_HOST_METRICS_THRESHOLD = 12 # months