add insights setting, optimize and consolidate queries

awx/api/urls/urls.py

@@ -32,7 +32,6 @@ from awx.api.views import (
     OAuth2TokenList,
     ApplicationOAuth2TokenList,
     OAuth2ApplicationDetail,
-    MetricsView,
 )
 
 from .organization import urls as organization_urls
@@ -132,7 +131,6 @@ v2_urls = [
     url(r'^applications/(?P<pk>[0-9]+)/tokens/$', ApplicationOAuth2TokenList.as_view(), name='application_o_auth2_token_list'),
     url(r'^tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'),
     url(r'^', include(oauth2_urls)),
-    url(r'^metrics/$', MetricsView.as_view(), name='metrics_view'),
 ]
 
 app_name = 'api'

awx/api/views/__init__.py

@@ -318,54 +318,6 @@ class DashboardJobsGraphView(APIView):
         return Response(dashboard_data)
 
 
-class MetricsView(APIView):
-
-    view_name = _("Metrics")
-    swagger_topic = 'Metrics'
-    
-    renderer_classes = [renderers.BrowsableAPIRenderer,
-                        renderers.PlainTextRenderer,
-                        JSONRenderer]
-    
-    def get(self, request, format=None):
-        ''' Show Metrics Details '''
-        
-        # Temporary Imports
-        from awx.main.models.organization import UserSessionMembership
-        from django.contrib.sessions.models import Session
-
-        def _prepare_data(data):
-            metrics = ''
-            for metric in data:
-                metrics += metric + '\n'
-            return metrics
-        
-        # Add active/expired, or only query active sessions
-        
-        total_sessions = Session.objects.all().count()
-        active_sessions = Session.objects.filter(expire_date__gte=now()).count()
-        
-        
-        api_sessions = UserSessionMembership.objects.all().count()
-        channels_sessions = total_sessions - api_sessions
-        expired_sessions = total_sessions - active_sessions
-        
-        data = []
-        data.append("# HELP awx_sessions_active counter A count of active sessions.")
-        data.append("# TYPE awx_sessions_active counter")
-        data.append("sessions.active_sessions {0} ".format(total_sessions))
-        data.append("# TYPE awx_sessions_websocket counter")
-        data.append("sessions.websocket_sessions {0} ".format(channels_sessions))
-        data.append("# TYPE awx_sessions_api counter")
-        data.append("sessions.api_sessions {0} ".format(api_sessions))
-        data.append("# TYPE sessions.active_sessions counter")
-        data.append("sessions.active_sessions {0}".format(active_sessions))
-        data.append("# TYPE sessions.expired_sessions counter")
-        data.append("sessions.expired_sessions {0}".format(expired_sessions))
-        
-        return Response(_prepare_data(data))
-
-
 class InstanceList(ListAPIView):
 
     view_name = _("Instances")

awx/api/views/root.py

@@ -103,7 +103,6 @@ class ApiVersionRootView(APIView):
             data['credential_types'] = reverse('api:credential_type_list', request=request)
             data['applications'] = reverse('api:o_auth2_application_list', request=request)
             data['tokens'] = reverse('api:o_auth2_token_list', request=request)
-            data['metrics'] = reverse('api:metrics_view', request=request)
         data['inventory'] = reverse('api:inventory_list', request=request)
         data['inventory_scripts'] = reverse('api:inventory_script_list', request=request)
         data['inventory_sources'] = reverse('api:inventory_source_list', request=request)
@@ -158,7 +157,6 @@ class ApiV1PingView(APIView):
             'ha': is_ha_environment(),
             'version': get_awx_version(),
             'active_node': settings.CLUSTER_HOST_ID,
-            'system_uuid': settings.system_uuid,
         }
 
         response['instances'] = []
@@ -278,3 +276,6 @@ class ApiV1ConfigView(APIView):
         except Exception:
             # FIX: Log
             return Response({"error": _("Failed to remove license.")}, status=status.HTTP_400_BAD_REQUEST)
+
+
+

awx/main/analytics/collectors.py

@@ -12,6 +12,7 @@ from django.contrib.sessions.models import Session
 from awx.main.analytics import register
 
 
+
 #
 # This module is used to define metrics collected by awx.main.analytics.gather()
 # Each function is decorated with a key name, and should return a data
@@ -28,6 +29,7 @@ from awx.main.analytics import register
 # data _since_ the last report date - i.e., new data in the last 24 hours)
 #
 
+
 @register('config')
 def config(since):
     license_info = get_license(show_key=False)
@@ -43,7 +45,6 @@ def config(since):
         'logging_aggregators': settings.LOG_AGGREGATOR_LOGGERS
     }
 
-
 @register('counts')
 def counts(since):
     counts = {}
@@ -77,23 +78,21 @@ def counts(since):
 @register('org_counts')
 def org_counts(since):
     counts = {}
-    for org in models.Organization.objects.annotate(
+    for org in models.Organization.objects.annotate(num_users=Count('member_role__members', distinct=True), 
-            num_users=Count('member_role__members', distinct=True), 
+                                                    num_teams=Count('teams', distinct=True)):  # Use .values to make a dict of only the fields we can about where
-            num_teams=Count('teams', distinct=True)):  # use .only()
         counts[org.id] = {'name': org.name,
-                          'users': org.num_users,
+                            'users': org.num_users,
-                          'teams': org.num_teams
+                            'teams': org.num_teams
-                          }
+                            }
     return counts
     
     
 @register('cred_type_counts')
 def cred_type_counts(since):
     counts = {}
-    for cred_type in models.CredentialType.objects.annotate(
+    for cred_type in models.CredentialType.objects.annotate(num_credentials=Count('credentials', distinct=True)):  
-            num_credentials=Count('credentials', distinct=True)):  
         counts[cred_type.id] = {'name': cred_type.name,
-                                'credential_count': cred_type.num_credentials
+                                  'credential_count': cred_type.num_credentials
                                 }
     return counts
     
@@ -102,14 +101,13 @@ def cred_type_counts(since):
 def inventory_counts(since):
     counts = {}
     from django.db.models import Count
-    for inv in models.Inventory.objects.annotate(
+    for inv in models.Inventory.objects.annotate(num_sources=Count('inventory_sources', distinct=True), 
-            num_sources=Count('inventory_sources', distinct=True), 
+                                                 num_hosts=Count('hosts', distinct=True)).only('id', 'name', 'kind'):
-            num_hosts=Count('hosts', distinct=True)).only('id', 'name', 'kind'):
         counts[inv.id] = {'name': inv.name,
-                          'kind': inv.kind,
+                            'kind': inv.kind,
-                          'hosts': inv.num_hosts,
+                            'hosts': inv.num_hosts,
-                          'sources': inv.num_sources
+                            'sources': inv.num_sources
-                          }
+                            }
     return counts
 
 
@@ -126,48 +124,24 @@ def projects_by_scm_type(since):
     return counts
 
 
-@register('job_counts')   # TODO: evaluate if we want this (was not an ask)  Also, may think about annotating rather than grabbing objects for efficiency 
+@register('job_counts')
-def job_counts(since):    # TODO: Optimize -- for example, all of these are going to need to be restrained to the last 24 hours/INSIGHTS_SCHEDULE
+def job_counts(since):    #TODO: Optimize -- for example, all of these are going to need to be restrained to the last 24 hours/INSIGHTS_SCHEDULE
     counts = {}
     counts['total_jobs'] = models.UnifiedJob.objects.all().count()
-    counts['successful_jobs'] = models.UnifiedJob.objects.filter(status='successful').count()
+    counts['status'] = dict(models.UnifiedJob.objects.values_list('status').annotate(Count('status')))
-    counts['cancelled_jobs'] = models.UnifiedJob.objects.filter(status='canceled').count()
-    counts['failed_jobs'] = models.UnifiedJob.objects.filter(status='failed').count()               
-    counts['error_jobs'] = models.UnifiedJob.objects.filter(status='error').count()   # Do we also want to include error, new, pending, waiting, running?
-    counts['new_jobs'] = models.UnifiedJob.objects.filter(status='new').count()       # Also, how much of this do we want `per instance`
-    counts['pending_jobs'] = models.UnifiedJob.objects.filter(status='pending').count()
-    counts['waiting_jobs'] = models.UnifiedJob.objects.filter(status='waiting').count()
-    counts['running_jobs'] = models.UnifiedJob.objects.filter(status='running').count()
-                                
-        
-    # These will later be used to optimize the jobs_running and jobs_total python properties ^^
-    # jobs_running = models.UnifiedJob.objects.filter(execution_node=instance, status__in=('running', 'waiting',)).count()
-    # jobs_total = models.UnifiedJob.objects.filter(execution_node=instance).count()
-        
-    return counts
-    
-    
-@register('job_counts_instance')
-def job_counts_instance(since):
-    counts = {}
     for instance in models.Instance.objects.all():
         counts[instance.id] = {'uuid': instance.uuid,
-                               'jobs_total': instance.jobs_total,       # this is _all_ jobs run by that node
+                                'jobs_total': models.UnifiedJob.objects.filter(execution_node=instance.hostname, status__in=('running', 'waiting',)).count(),
-                               'jobs_running': instance.jobs_running,   # this is jobs in running & waiting state
+                                'jobs_running': models.UnifiedJob.objects.filter(execution_node=instance.hostname).count(),   # jobs in running & waiting state
-                               'launch_type': {'manual': models.UnifiedJob.objects.filter(launch_type='manual').count(), # I can definitely condense this
+                                'launch_type': dict(models.UnifiedJob.objects.filter(execution_node=instance.hostname).values_list('launch_type').annotate(Count('launch_type')))
-                                               'relaunch': models.UnifiedJob.objects.filter(launch_type='relaunch').count(),
+                                }
-                                               'scheduled': models.UnifiedJob.objects.filter(launch_type='scheduled').count(),
-                                               'callback': models.UnifiedJob.objects.filter(launch_type='callback').count(),
-                                               'dependency': models.UnifiedJob.objects.filter(launch_type='dependency').count(),
-                                               'sync': models.UnifiedJob.objects.filter(launch_type='workflow').count(),
-                                               'scm': models.UnifiedJob.objects.filter(launch_type='scm').count()
-                                               }
-                               }
     return counts
     
     
+    
 @register('jobs')
 def jobs(since):
     counts = {}
+    jobs = models.Job.objects.filter(created__gt=since)
     counts['latest_jobs'] = models.Job.objects.filter(created__gt=since).count()
     return counts

awx/main/analytics/core.py

@@ -31,7 +31,7 @@ def _valid_license():
         logger.exception("A valid license was not found:")
         return False
     return True
-
+    
 
 def register(key):
     """
@@ -73,6 +73,10 @@ def gather(dest=None, module=None):
     if last_run < max_interval or not last_run:
         last_run = max_interval
 
+    if settings.INSIGHTS_DATA_ENABLED:
+        logger.exception("Insights not enabled.  Analytics data not gathered.")
+        return
+
     if _valid_license() is False:
         logger.exception("Invalid License provided, or No License Provided")
         return

awx/main/conf.py

@@ -301,6 +301,16 @@ register(
     placeholder={'HTTP_PROXY': 'myproxy.local:8080'},
 )
 
+register(
+    'INSIGHTS_DATA_ENABLED',
+    field_class=fields.BooleanField,
+    default=False,
+    label=_('Insights Analytics Data'),
+    help_text=_('Enables Tower to gather analytics data about itself and send it to Insights.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
 register(
     'AWX_ROLES_ENABLED',
     field_class=fields.BooleanField,

awx/main/migrations/0064_v350_analytics_state.py

@@ -8,7 +8,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('main', '0062_v350_new_playbook_stats'),
+        ('main', '0063_v350_org_host_limits'),
     ]
 
     operations = [

awx/settings/defaults.py

@@ -672,6 +672,12 @@ AWX_AUTO_DEPROVISION_INSTANCES = False
 # Note: This setting may be overridden by database settings.
 PENDO_TRACKING_STATE = "off"
 
+# Enables Insights data collection for Ansible Tower.  Defaults to the value of PENDO_TRACKING_STATE
+if PENDO_TRACKING_STATE == "off":
+    INSIGHTS_DATA_ENABLED = False
+else:
+    INSIGHTS_DATA_ENABLED = True
+
 # Default list of modules allowed for ad hoc commands.
 # Note: This setting may be overridden by database settings.
 AD_HOC_COMMANDS = [

awx/settings/development.py

@@ -88,6 +88,7 @@ AWX_ISOLATED_LAUNCH_TIMEOUT = 30
 # Disable Pendo on the UI for development/test.
 # Note: This setting may be overridden by database settings.
 PENDO_TRACKING_STATE = "off"
+INSIGHTS_DATA_ENABLED = False
 
 # Use Django-Jenkins if installed. Only run tests for awx.main app.
 try: