Address the problems with trying to use a JSONField

awx/main/analytics/collectors.py

@@ -1,11 +1,12 @@
 import io
+import json
 import os
 import os.path
 import platform
 import distro
 
 from django.db import connection
-from django.db.models import Count, Max
+from django.db.models import Count, Max, Min
 from django.conf import settings
 from django.utils.timezone import now, timedelta
 from django.utils.translation import ugettext_lazy as _
@@ -295,11 +296,13 @@ def _copy_table(table, query, path):
 def events_slicing(key, since, until):
     from awx.conf.models import Setting
 
+    pk_values = models.JobEvent.objects.filter(created__gte=since, created__lte=until).aggregate(Min('pk'), Max('pk'))
+
     step = 100000
     last_entries = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_ENTRIES').first()
-    last_entries = last_entries.value if last_entries is not None else {}
+    last_entries = json.loads((last_entries.value if last_entries is not None else '') or '{}')
-    previous_pk = last_entries.get(key) or 0
+    previous_pk = last_entries.get(key) or pk_values['pk__min'] or 0
-    final_pk = models.JobEvent.objects.filter(created_lte=until).aggregate(Max('pk'))['pk__max']
+    final_pk = pk_values['pk__max']
 
     for start in range(previous_pk, final_pk + 1, step):
         yield (start, min(start + step, final_pk))

awx/main/analytics/core.py

@@ -141,7 +141,7 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti
             until = now()
         last_run = since or settings.AUTOMATION_ANALYTICS_LAST_GATHER or (until - timedelta(weeks=4))
         last_entries = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_ENTRIES').first()
-        last_entries = last_entries.value if last_entries is not None else {}
+        last_entries = json.loads((last_entries.value if last_entries is not None else '') or '{}')
         logger.debug("Last analytics run was: {}".format(settings.AUTOMATION_ANALYTICS_LAST_GATHER))
 
         collector_module = module if module else collectors
@@ -151,7 +151,9 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti
             if inspect.isfunction(func) and hasattr(func, '__awx_analytics_key__') and (not subset or name in subset)
         ]
         if collection_type != 'dry-run' and not any(c.__awx_analytics_key__ == 'config' for c in collector_list):
+            # In order to ship to analytics, we must include the output of the built-in 'config' collector.
             collector_list.append(collectors.config)
+
         json_collectors = [func for func in collector_list if func.__awx_analytics_type__ == 'json']
         csv_collectors = [func for func in collector_list if func.__awx_analytics_type__ == 'csv']
 
@@ -182,7 +184,7 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti
                     with disable_activity_stream():
                         for filename in data:
                             last_entries[filename.replace('.json', '')] = until
-                        settings.AUTOMATION_ANALYTICS_LAST_ENTRIES = last_entries
+                        settings.AUTOMATION_ANALYTICS_LAST_ENTRIES = json.dumps(last_entries)
 
         for func in csv_collectors:
             key = func.__awx_analytics_key__
@@ -211,7 +213,7 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti
                                 ship(tgzfile)
                                 with disable_activity_stream():
                                     last_entries[key] = end
-                                    settings.AUTOMATION_ANALYTICS_LAST_ENTRIES = last_entries
+                                    settings.AUTOMATION_ANALYTICS_LAST_ENTRIES = json.dumps(last_entries)
             except Exception:
                 logger.exception("Could not generate metric {}".format(filename))
 

awx/main/conf.py

@@ -10,7 +10,6 @@ from rest_framework.fields import FloatField
 
 # Tower
 from awx.conf import fields, register, register_validate
-from awx.main.fields import JSONField
 from awx.main.models import ExecutionEnvironment
 
 
@@ -781,8 +780,10 @@ register(
 )
 register(
     'AUTOMATION_ANALYTICS_LAST_ENTRIES',
-    field_class=JSONField,
+    field_class=fields.CharField,
-    # label=_('Last gathered entries for expensive Automation Analytics collectors.'),
+    label=_('Last gathered entries for expensive Automation Analytics collectors.'),
+    default='',
+    allow_blank=True,
     category=_('System'),
     category_slug='system',
 )

awx/settings/defaults.py

@@ -621,7 +621,7 @@ INSIGHTS_TRACKING_STATE = False
 # Last gather date for Analytics
 AUTOMATION_ANALYTICS_LAST_GATHER = None
 # Last gathered entries for expensive Analytics
-AUTOMATION_ANALYTICS_LAST_ENTRIES = None
+AUTOMATION_ANALYTICS_LAST_ENTRIES = ''
 
 # Default list of modules allowed for ad hoc commands.
 # Note: This setting may be overridden by database settings.