From 44db4587be0eead95355e244574818dddf35c8b7 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Mon, 3 Apr 2023 17:26:37 +0200 Subject: [PATCH 1/6] Analytics upload: HostMetrics hybrid sync --- awx/main/analytics/collectors.py | 76 ++++++++++++++++++++++++++++++-- awx/main/analytics/core.py | 22 +++++++-- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index e7655997d2..1bc4c9044f 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -6,7 +6,7 @@ import platform import distro from django.db import connection -from django.db.models import Count +from django.db.models import Count, Min from django.conf import settings from django.contrib.sessions.models import Session from django.utils.timezone import now, timedelta @@ -35,7 +35,7 @@ data _since_ the last report date - i.e., new data in the last 24 hours) """ -def trivial_slicing(key, since, until, last_gather): +def trivial_slicing(key, since, until, last_gather, **kwargs): if since is not None: return [(since, until)] @@ -48,7 +48,7 @@ def trivial_slicing(key, since, until, last_gather): return [(last_entry, until)] -def four_hour_slicing(key, since, until, last_gather): +def four_hour_slicing(key, since, until, last_gather, **kwargs): if since is not None: last_entry = since else: @@ -69,6 +69,54 @@ def four_hour_slicing(key, since, until, last_gather): start = end +def host_metric_slicing(key, since, until, last_gather, **kwargs): + """ + Slicing doesn't start 4 weeks ago, but sends whole table monthly or first time + """ + from awx.main.models.inventory import HostMetric + + if since is not None: + return [(since, until)] + + from awx.conf.models import Setting + + # Check if full sync should be done + full_sync_enabled = kwargs.get('full_sync_enabled', False) + last_entry = None + if not full_sync_enabled: + # + # If not, try incremental sync first + # + last_entries = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_ENTRIES').first() + last_entries = json.loads((last_entries.value if last_entries is not None else '') or '{}', object_hook=datetime_hook) + last_entry = last_entries.get(key) + if not last_entry: + # + # If not done before, switch to full sync + # + full_sync_enabled = True + + if full_sync_enabled: + # + # Find the lowest date for full sync + # + min_dates = HostMetric.objects.aggregate(min_last_automation=Min('last_automation'), min_last_deleted=Min('last_deleted')) + if min_dates['min_last_automation'] and min_dates['min_last_deleted']: + last_entry = min(min_dates['min_last_automation'], min_dates['min_last_deleted']) + elif min_dates['min_last_automation'] or min_dates['min_last_deleted']: + last_entry = min_dates['min_last_automation'] or min_dates['min_last_deleted'] + + if not last_entry: + # empty table + return [] + + start, end = last_entry, None + while start < until: + end = min(start + timedelta(days=30), until) + yield (start, end) + start = end + + def _identify_lower(key, since, until, last_gather): from awx.conf.models import Setting @@ -537,3 +585,25 @@ def workflow_job_template_node_table(since, full_path, **kwargs): ) always_nodes ON main_workflowjobtemplatenode.id = always_nodes.from_workflowjobtemplatenode_id ORDER BY main_workflowjobtemplatenode.id ASC) TO STDOUT WITH CSV HEADER''' return _copy_table(table='workflow_job_template_node', query=workflow_job_template_node_query, path=full_path) + + +@register( + 'host_metric_table', '1.0', format='csv', description=_('Host Metric data, incremental/full sync'), expensive=host_metric_slicing, full_sync_interval=30 +) +def host_metric_table(since, full_path, until, **kwargs): + host_metric_query = '''COPY (SELECT main_hostmetric.id, + main_hostmetric.hostname, + main_hostmetric.first_automation, + main_hostmetric.last_automation, + main_hostmetric.last_deleted, + main_hostmetric.deleted, + main_hostmetric.automated_counter, + main_hostmetric.deleted_counter, + main_hostmetric.used_in_inventories + FROM main_hostmetric + WHERE (main_hostmetric.last_automation > '{}' AND main_hostmetric.last_automation <= '{}') OR + (main_hostmetric.last_deleted > '{}' AND main_hostmetric.last_deleted <= '{}') + ORDER BY main_hostmetric.id ASC) TO STDOUT WITH CSV HEADER'''.format( + since.isoformat(), until.isoformat(), since.isoformat(), until.isoformat() + ) + return _copy_table(table='host_metric', query=host_metric_query, path=full_path) diff --git a/awx/main/analytics/core.py b/awx/main/analytics/core.py index 77f6108205..9cbc873b2b 100644 --- a/awx/main/analytics/core.py +++ b/awx/main/analytics/core.py @@ -52,7 +52,7 @@ def all_collectors(): } -def register(key, version, description=None, format='json', expensive=None): +def register(key, version, description=None, format='json', expensive=None, full_sync_interval=None): """ A decorator used to register a function as a metric collector. @@ -71,6 +71,7 @@ def register(key, version, description=None, format='json', expensive=None): f.__awx_analytics_description__ = description f.__awx_analytics_type__ = format f.__awx_expensive__ = expensive + f.__awx_full_sync_interval__ = full_sync_interval return f return decorate @@ -259,10 +260,19 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti # These slicer functions may return a generator. The `since` parameter is # allowed to be None, and will fall back to LAST_ENTRIES[key] or to # LAST_GATHER (truncated appropriately to match the 4-week limit). + # + # Or it can force full table sync if interval is given + kwargs = dict() + full_sync_enabled = False + if func.__awx_full_sync_interval__: + last_full_sync = last_entries.get(f"{key}_full") + full_sync_enabled = not last_full_sync or last_full_sync < now() - timedelta(days=func.__awx_full_sync_interval__) + + kwargs['full_sync_enabled'] = full_sync_enabled if func.__awx_expensive__: - slices = func.__awx_expensive__(key, since, until, last_gather) + slices = func.__awx_expensive__(key, since, until, last_gather, **kwargs) else: - slices = collectors.trivial_slicing(key, since, until, last_gather) + slices = collectors.trivial_slicing(key, since, until, last_gather, **kwargs) for start, end in slices: files = func(start, full_path=gather_dir, until=end) @@ -301,6 +311,12 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti succeeded = False logger.exception("Could not generate metric {}".format(filename)) + # update full sync timestamp if successfully shipped + if full_sync_enabled and collection_type != 'dry-run' and succeeded: + with disable_activity_stream(): + last_entries[f"{key}_full"] = now() + settings.AUTOMATION_ANALYTICS_LAST_ENTRIES = json.dumps(last_entries, cls=DjangoJSONEncoder) + if collection_type != 'dry-run': if succeeded: for fpath in tarfiles: From fff6fa7d7a7e61aa7d1a60757002bb5a7a1cb982 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Thu, 23 Mar 2023 14:16:20 +0100 Subject: [PATCH 2/6] Additional Licensing values --- awx/main/utils/licensing.py | 6 +++++- .../SubscriptionDetail/SubscriptionDetail.js | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/awx/main/utils/licensing.py b/awx/main/utils/licensing.py index b3ea7723e5..c692e3131a 100644 --- a/awx/main/utils/licensing.py +++ b/awx/main/utils/licensing.py @@ -388,9 +388,13 @@ class Licenser(object): if subscription_model == SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS: automated_instances = HostMetric.active_objects.count() first_host = HostMetric.active_objects.only('first_automation').order_by('first_automation').first() + attrs['deleted_instances'] = HostMetric.objects.filter(deleted=True).count() + attrs['reactivated_instances'] = HostMetric.active_objects.filter(deleted_counter__gte=1).count() else: - automated_instances = HostMetric.objects.count() + automated_instances = 0 first_host = HostMetric.objects.only('first_automation').order_by('first_automation').first() + attrs['deleted_instances'] = 0 + attrs['reactivated_instances'] = 0 if first_host: automated_since = int(first_host.first_automation.timestamp()) diff --git a/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js b/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js index a9349e3345..bbb4935c6f 100644 --- a/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js +++ b/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js @@ -112,6 +112,16 @@ function SubscriptionDetail() { label={t`Hosts remaining`} value={license_info.free_instances} /> + + {license_info.instance_count < 9999999 && ( Date: Mon, 27 Mar 2023 12:06:22 +0200 Subject: [PATCH 3/6] HostMetric Cleanup task --- awx/main/conf.py | 9 ++++++++ .../commands/cleanup_host_metrics.py | 22 +++++++++++++++++++ awx/main/models/inventory.py | 12 ++++++++++ awx/main/tasks/system.py | 15 +++++++++++++ awx/settings/defaults.py | 8 +++++++ 5 files changed, 66 insertions(+) create mode 100644 awx/main/management/commands/cleanup_host_metrics.py diff --git a/awx/main/conf.py b/awx/main/conf.py index 33b3b4714d..c9515e845e 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -822,6 +822,15 @@ register( category_slug='system', ) +register( + 'CLEANUP_HOST_METRICS_LAST_TS', + field_class=fields.DateTimeField, + label=_('Last cleanup date for HostMetrics'), + allow_null=True, + category=_('System'), + category_slug='system', +) + def logging_validate(serializer, attrs): if not serializer.instance or not hasattr(serializer.instance, 'LOG_AGGREGATOR_HOST') or not hasattr(serializer.instance, 'LOG_AGGREGATOR_TYPE'): diff --git a/awx/main/management/commands/cleanup_host_metrics.py b/awx/main/management/commands/cleanup_host_metrics.py new file mode 100644 index 0000000000..788ffcba94 --- /dev/null +++ b/awx/main/management/commands/cleanup_host_metrics.py @@ -0,0 +1,22 @@ +from awx.main.models import HostMetric +from django.core.management.base import BaseCommand +from django.conf import settings + + +class Command(BaseCommand): + """ + Run soft-deleting of HostMetrics + """ + + help = 'Run soft-deleting of HostMetrics' + + def add_arguments(self, parser): + parser.add_argument('--months-ago', type=int, dest='months-ago', action='store', help='Threshold in months for soft-deleting') + + def handle(self, *args, **options): + months_ago = options.get('months-ago') or None + + if not months_ago: + months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_THRESHOLD', 12) + + HostMetric.cleanup_task(months_ago) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index fe1b09e23d..626775a9d6 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -9,6 +9,8 @@ import re import copy import os.path from urllib.parse import urljoin + +import dateutil.relativedelta import yaml # Django @@ -888,6 +890,16 @@ class HostMetric(models.Model): self.deleted = False self.save(update_fields=['deleted']) + @classmethod + def cleanup_task(cls, months_ago): + last_automation_before = now() - dateutil.relativedelta.relativedelta(months=months_ago) + + logger.info(f'Cleanup [HostMetric]: soft-deleting records last automated before {last_automation_before}') + HostMetric.objects.filter(last_automation__lt=last_automation_before).update( + deleted=True, deleted_counter=models.F('deleted_counter') + 1, last_deleted=now() + ) + settings.CLEANUP_HOST_METRICS_LAST_TS = now() + class HostMetricSummaryMonthly(models.Model): """ diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index a83dad5ccc..36fc266803 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -47,6 +47,7 @@ from awx.main.models import ( Inventory, SmartInventoryMembership, Job, + HostMetric, ) from awx.main.constants import ACTIVE_STATES from awx.main.dispatch.publish import task @@ -378,6 +379,20 @@ def cleanup_images_and_files(): _cleanup_images_and_files() +@task(queue=get_task_queuename) +def cleanup_host_metrics(): + from awx.conf.models import Setting + from rest_framework.fields import DateTimeField + + last_cleanup = Setting.objects.filter(key='CLEANUP_HOST_METRICS_LAST_TS').first() + last_time = DateTimeField().to_internal_value(last_cleanup.value) if last_cleanup and last_cleanup.value else None + + cleanup_interval_secs = getattr(settings, 'CLEANUP_HOST_METRICS_INTERVAL', 30) * 86400 + if not last_time or ((now() - last_time).total_seconds() > cleanup_interval_secs): + months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_THRESHOLD', 12) + HostMetric.cleanup_task(months_ago) + + @task(queue=get_task_queuename) def cluster_node_health_check(node): """ diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 593e6ae002..82d0e7b343 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -475,6 +475,7 @@ CELERYBEAT_SCHEDULE = { 'receptor_reaper': {'task': 'awx.main.tasks.system.awx_receptor_workunit_reaper', 'schedule': timedelta(seconds=60)}, 'send_subsystem_metrics': {'task': 'awx.main.analytics.analytics_tasks.send_subsystem_metrics', 'schedule': timedelta(seconds=20)}, 'cleanup_images': {'task': 'awx.main.tasks.system.cleanup_images_and_files', 'schedule': timedelta(hours=3)}, + 'cleanup_host_metrics': {'task': 'awx.main.tasks.system.cleanup_host_metrics', 'schedule': timedelta(days=1)}, } # Django Caching Configuration @@ -1052,3 +1053,10 @@ UI_NEXT = True # - '': No model - Subscription not counted from Host Metrics # - 'unique_managed_hosts': Compliant = automated - deleted hosts (using /api/v2/host_metrics/) SUBSCRIPTION_USAGE_MODEL = '' + +# Host metrics cleanup - last time of the cleanup run (soft-deleting records) +CLEANUP_HOST_METRICS_LAST_TS = None +# Host metrics cleanup - minimal interval between two cleanups in days +CLEANUP_HOST_METRICS_INTERVAL = 30 # days +# Host metrics cleanup - soft-delete HostMetric records with last_automation < [threshold] (in months) +CLEANUP_HOST_METRICS_THRESHOLD = 12 # months From 64923e12fcc6a898b3352dc593d72e626e4f147a Mon Sep 17 00:00:00 2001 From: Salma Kochay Date: Wed, 29 Mar 2023 13:42:25 -0400 Subject: [PATCH 4/6] show/hide host metric subscription details --- .../SubscriptionDetail/SubscriptionDetail.js | 92 ++++++++++--------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js b/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js index bbb4935c6f..70356a030a 100644 --- a/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js +++ b/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js @@ -24,7 +24,7 @@ const HelperText = styled(PFHelperText)` `; function SubscriptionDetail() { - const { me = {}, license_info, version } = useConfig(); + const { me = {}, license_info, version, systemConfig } = useConfig(); const baseURL = '/settings/subscription'; const tabsArray = [ { @@ -56,35 +56,37 @@ function SubscriptionDetail() { - - - - {t`The number of hosts you have automated against is below your subscription count.`} - - - ) : ( - <> - - - {t`You have automated against more hosts than your subscription allows.`} - - - ) - } - /> + {systemConfig?.SUBSCRIPTION_USAGE_MODEL === 'unique_managed_hosts' && ( + + + + {t`The number of hosts you have automated against is below your subscription count.`} + + + ) : ( + <> + + + {t`You have automated against more hosts than your subscription allows.`} + + + ) + } + /> + )} {typeof automatedInstancesCount !== 'undefined' && automatedInstancesCount !== null && ( - - - + )} + {systemConfig?.SUBSCRIPTION_USAGE_MODEL === 'unique_managed_hosts' && ( + + )} + {systemConfig?.SUBSCRIPTION_USAGE_MODEL === 'unique_managed_hosts' && ( + + /> + )} {license_info.instance_count < 9999999 && ( Date: Mon, 3 Apr 2023 13:27:14 -0400 Subject: [PATCH 5/6] UI test fixes for hiding subscription details --- .../SubscriptionDetail/SubscriptionDetail.js | 12 ++++++++---- .../SubscriptionDetail/SubscriptionDetail.test.js | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js b/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js index 70356a030a..6f16cbbb4d 100644 --- a/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js +++ b/awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js @@ -56,7 +56,8 @@ function SubscriptionDetail() { - {systemConfig?.SUBSCRIPTION_USAGE_MODEL === 'unique_managed_hosts' && ( + {systemConfig?.SUBSCRIPTION_USAGE_MODEL === + 'unique_managed_hosts' && ( - {systemConfig?.SUBSCRIPTION_USAGE_MODEL === 'unique_managed_hosts' && ( + {systemConfig?.SUBSCRIPTION_USAGE_MODEL === + 'unique_managed_hosts' && ( )} - {systemConfig?.SUBSCRIPTION_USAGE_MODEL === 'unique_managed_hosts' && ( + {systemConfig?.SUBSCRIPTION_USAGE_MODEL === + 'unique_managed_hosts' && ( )} - {systemConfig?.SUBSCRIPTION_USAGE_MODEL === 'unique_managed_hosts' && ( + {systemConfig?.SUBSCRIPTION_USAGE_MODEL === + 'unique_managed_hosts' && ( ', () => { From 20817789bd0f20f327ff8d286e7237a68f021b06 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Wed, 5 Apr 2023 09:54:36 +0200 Subject: [PATCH 6/6] HostMetric task param check --- awx/main/models/inventory.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 626775a9d6..21b3839461 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -892,13 +892,20 @@ class HostMetric(models.Model): @classmethod def cleanup_task(cls, months_ago): - last_automation_before = now() - dateutil.relativedelta.relativedelta(months=months_ago) + try: + months_ago = int(months_ago) + if months_ago <= 0: + raise ValueError() - logger.info(f'Cleanup [HostMetric]: soft-deleting records last automated before {last_automation_before}') - HostMetric.objects.filter(last_automation__lt=last_automation_before).update( - deleted=True, deleted_counter=models.F('deleted_counter') + 1, last_deleted=now() - ) - settings.CLEANUP_HOST_METRICS_LAST_TS = now() + last_automation_before = now() - dateutil.relativedelta.relativedelta(months=months_ago) + + logger.info(f'Cleanup [HostMetric]: soft-deleting records last automated before {last_automation_before}') + HostMetric.active_objects.filter(last_automation__lt=last_automation_before).update( + deleted=True, deleted_counter=models.F('deleted_counter') + 1, last_deleted=now() + ) + settings.CLEANUP_HOST_METRICS_LAST_TS = now() + except (TypeError, ValueError): + logger.error(f"Cleanup [HostMetric]: months_ago({months_ago}) has to be a positive integer value") class HostMetricSummaryMonthly(models.Model):