diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index 2244d65ac5..2ca217c8d7 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -1,11 +1,12 @@ import io +import json import os import os.path import platform import distro from django.db import connection -from django.db.models import Count, Max +from django.db.models import Count, Max, Min from django.conf import settings from django.utils.timezone import now, timedelta from django.utils.translation import ugettext_lazy as _ @@ -295,11 +296,13 @@ def _copy_table(table, query, path): def events_slicing(key, since, until): from awx.conf.models import Setting + pk_values = models.JobEvent.objects.filter(created__gte=since, created__lte=until).aggregate(Min('pk'), Max('pk')) + step = 100000 last_entries = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_ENTRIES').first() - last_entries = last_entries.value if last_entries is not None else {} - previous_pk = last_entries.get(key) or 0 - final_pk = models.JobEvent.objects.filter(created_lte=until).aggregate(Max('pk'))['pk__max'] + last_entries = json.loads((last_entries.value if last_entries is not None else '') or '{}') + previous_pk = last_entries.get(key) or pk_values['pk__min'] or 0 + final_pk = pk_values['pk__max'] for start in range(previous_pk, final_pk + 1, step): yield (start, min(start + step, final_pk)) diff --git a/awx/main/analytics/core.py b/awx/main/analytics/core.py index 938c110daf..7abcd76b9f 100644 --- a/awx/main/analytics/core.py +++ b/awx/main/analytics/core.py @@ -141,7 +141,7 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti until = now() last_run = since or settings.AUTOMATION_ANALYTICS_LAST_GATHER or (until - timedelta(weeks=4)) last_entries = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_ENTRIES').first() - last_entries = last_entries.value if last_entries is not None else {} + last_entries = json.loads((last_entries.value if last_entries is not None else '') or '{}') logger.debug("Last analytics run was: {}".format(settings.AUTOMATION_ANALYTICS_LAST_GATHER)) collector_module = module if module else collectors @@ -151,7 +151,9 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti if inspect.isfunction(func) and hasattr(func, '__awx_analytics_key__') and (not subset or name in subset) ] if collection_type != 'dry-run' and not any(c.__awx_analytics_key__ == 'config' for c in collector_list): + # In order to ship to analytics, we must include the output of the built-in 'config' collector. collector_list.append(collectors.config) + json_collectors = [func for func in collector_list if func.__awx_analytics_type__ == 'json'] csv_collectors = [func for func in collector_list if func.__awx_analytics_type__ == 'csv'] @@ -182,7 +184,7 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti with disable_activity_stream(): for filename in data: last_entries[filename.replace('.json', '')] = until - settings.AUTOMATION_ANALYTICS_LAST_ENTRIES = last_entries + settings.AUTOMATION_ANALYTICS_LAST_ENTRIES = json.dumps(last_entries) for func in csv_collectors: key = func.__awx_analytics_key__ @@ -211,7 +213,7 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti ship(tgzfile) with disable_activity_stream(): last_entries[key] = end - settings.AUTOMATION_ANALYTICS_LAST_ENTRIES = last_entries + settings.AUTOMATION_ANALYTICS_LAST_ENTRIES = json.dumps(last_entries) except Exception: logger.exception("Could not generate metric {}".format(filename)) diff --git a/awx/main/conf.py b/awx/main/conf.py index 3699d3f685..5c5df72218 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -10,7 +10,6 @@ from rest_framework.fields import FloatField # Tower from awx.conf import fields, register, register_validate -from awx.main.fields import JSONField from awx.main.models import ExecutionEnvironment @@ -781,8 +780,10 @@ register( ) register( 'AUTOMATION_ANALYTICS_LAST_ENTRIES', - field_class=JSONField, - # label=_('Last gathered entries for expensive Automation Analytics collectors.'), + field_class=fields.CharField, + label=_('Last gathered entries for expensive Automation Analytics collectors.'), + default='', + allow_blank=True, category=_('System'), category_slug='system', ) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 1a91d392c7..93ad306b1e 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -621,7 +621,7 @@ INSIGHTS_TRACKING_STATE = False # Last gather date for Analytics AUTOMATION_ANALYTICS_LAST_GATHER = None # Last gathered entries for expensive Analytics -AUTOMATION_ANALYTICS_LAST_ENTRIES = None +AUTOMATION_ANALYTICS_LAST_ENTRIES = '' # Default list of modules allowed for ad hoc commands. # Note: This setting may be overridden by database settings.