From 636153d92c183f52576fc1d26fb6032613cd8167 Mon Sep 17 00:00:00 2001 From: Christian Adams Date: Wed, 6 Mar 2019 10:54:02 -0500 Subject: [PATCH] add insights setting, optimize and consolidate queries --- awx/api/urls/urls.py | 2 - awx/api/views/__init__.py | 48 ------------ awx/api/views/root.py | 5 +- awx/main/analytics/collectors.py | 74 ++++++------------- awx/main/analytics/core.py | 6 +- awx/main/conf.py | 10 +++ ..._state.py => 0064_v350_analytics_state.py} | 2 +- awx/settings/defaults.py | 6 ++ awx/settings/development.py | 1 + 9 files changed, 50 insertions(+), 104 deletions(-) rename awx/main/migrations/{0063_v350_analytics_state.py => 0064_v350_analytics_state.py} (92%) diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index 4a06a82531..52e9ef1cf0 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -32,7 +32,6 @@ from awx.api.views import ( OAuth2TokenList, ApplicationOAuth2TokenList, OAuth2ApplicationDetail, - MetricsView, ) from .organization import urls as organization_urls @@ -132,7 +131,6 @@ v2_urls = [ url(r'^applications/(?P[0-9]+)/tokens/$', ApplicationOAuth2TokenList.as_view(), name='application_o_auth2_token_list'), url(r'^tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'), url(r'^', include(oauth2_urls)), - url(r'^metrics/$', MetricsView.as_view(), name='metrics_view'), ] app_name = 'api' diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 89c6c7bb4b..902c7d6910 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -318,54 +318,6 @@ class DashboardJobsGraphView(APIView): return Response(dashboard_data) -class MetricsView(APIView): - - view_name = _("Metrics") - swagger_topic = 'Metrics' - - renderer_classes = [renderers.BrowsableAPIRenderer, - renderers.PlainTextRenderer, - JSONRenderer] - - def get(self, request, format=None): - ''' Show Metrics Details ''' - - # Temporary Imports - from awx.main.models.organization import UserSessionMembership - from django.contrib.sessions.models import Session - - def _prepare_data(data): - metrics = '' - for metric in data: - metrics += metric + '\n' - return metrics - - # Add active/expired, or only query active sessions - - total_sessions = Session.objects.all().count() - active_sessions = Session.objects.filter(expire_date__gte=now()).count() - - - api_sessions = UserSessionMembership.objects.all().count() - channels_sessions = total_sessions - api_sessions - expired_sessions = total_sessions - active_sessions - - data = [] - data.append("# HELP awx_sessions_active counter A count of active sessions.") - data.append("# TYPE awx_sessions_active counter") - data.append("sessions.active_sessions {0} ".format(total_sessions)) - data.append("# TYPE awx_sessions_websocket counter") - data.append("sessions.websocket_sessions {0} ".format(channels_sessions)) - data.append("# TYPE awx_sessions_api counter") - data.append("sessions.api_sessions {0} ".format(api_sessions)) - data.append("# TYPE sessions.active_sessions counter") - data.append("sessions.active_sessions {0}".format(active_sessions)) - data.append("# TYPE sessions.expired_sessions counter") - data.append("sessions.expired_sessions {0}".format(expired_sessions)) - - return Response(_prepare_data(data)) - - class InstanceList(ListAPIView): view_name = _("Instances") diff --git a/awx/api/views/root.py b/awx/api/views/root.py index bd64dc4aa4..c7ecbbeef5 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -103,7 +103,6 @@ class ApiVersionRootView(APIView): data['credential_types'] = reverse('api:credential_type_list', request=request) data['applications'] = reverse('api:o_auth2_application_list', request=request) data['tokens'] = reverse('api:o_auth2_token_list', request=request) - data['metrics'] = reverse('api:metrics_view', request=request) data['inventory'] = reverse('api:inventory_list', request=request) data['inventory_scripts'] = reverse('api:inventory_script_list', request=request) data['inventory_sources'] = reverse('api:inventory_source_list', request=request) @@ -158,7 +157,6 @@ class ApiV1PingView(APIView): 'ha': is_ha_environment(), 'version': get_awx_version(), 'active_node': settings.CLUSTER_HOST_ID, - 'system_uuid': settings.system_uuid, } response['instances'] = [] @@ -278,3 +276,6 @@ class ApiV1ConfigView(APIView): except Exception: # FIX: Log return Response({"error": _("Failed to remove license.")}, status=status.HTTP_400_BAD_REQUEST) + + + diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index b0462221e0..ee4719a98f 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -12,6 +12,7 @@ from django.contrib.sessions.models import Session from awx.main.analytics import register + # # This module is used to define metrics collected by awx.main.analytics.gather() # Each function is decorated with a key name, and should return a data @@ -28,6 +29,7 @@ from awx.main.analytics import register # data _since_ the last report date - i.e., new data in the last 24 hours) # + @register('config') def config(since): license_info = get_license(show_key=False) @@ -43,7 +45,6 @@ def config(since): 'logging_aggregators': settings.LOG_AGGREGATOR_LOGGERS } - @register('counts') def counts(since): counts = {} @@ -77,23 +78,21 @@ def counts(since): @register('org_counts') def org_counts(since): counts = {} - for org in models.Organization.objects.annotate( - num_users=Count('member_role__members', distinct=True), - num_teams=Count('teams', distinct=True)): # use .only() + for org in models.Organization.objects.annotate(num_users=Count('member_role__members', distinct=True), + num_teams=Count('teams', distinct=True)): # Use .values to make a dict of only the fields we can about where counts[org.id] = {'name': org.name, - 'users': org.num_users, - 'teams': org.num_teams - } + 'users': org.num_users, + 'teams': org.num_teams + } return counts @register('cred_type_counts') def cred_type_counts(since): counts = {} - for cred_type in models.CredentialType.objects.annotate( - num_credentials=Count('credentials', distinct=True)): + for cred_type in models.CredentialType.objects.annotate(num_credentials=Count('credentials', distinct=True)): counts[cred_type.id] = {'name': cred_type.name, - 'credential_count': cred_type.num_credentials + 'credential_count': cred_type.num_credentials } return counts @@ -102,14 +101,13 @@ def cred_type_counts(since): def inventory_counts(since): counts = {} from django.db.models import Count - for inv in models.Inventory.objects.annotate( - num_sources=Count('inventory_sources', distinct=True), - num_hosts=Count('hosts', distinct=True)).only('id', 'name', 'kind'): + for inv in models.Inventory.objects.annotate(num_sources=Count('inventory_sources', distinct=True), + num_hosts=Count('hosts', distinct=True)).only('id', 'name', 'kind'): counts[inv.id] = {'name': inv.name, - 'kind': inv.kind, - 'hosts': inv.num_hosts, - 'sources': inv.num_sources - } + 'kind': inv.kind, + 'hosts': inv.num_hosts, + 'sources': inv.num_sources + } return counts @@ -126,48 +124,24 @@ def projects_by_scm_type(since): return counts -@register('job_counts') # TODO: evaluate if we want this (was not an ask) Also, may think about annotating rather than grabbing objects for efficiency -def job_counts(since): # TODO: Optimize -- for example, all of these are going to need to be restrained to the last 24 hours/INSIGHTS_SCHEDULE +@register('job_counts') +def job_counts(since): #TODO: Optimize -- for example, all of these are going to need to be restrained to the last 24 hours/INSIGHTS_SCHEDULE counts = {} counts['total_jobs'] = models.UnifiedJob.objects.all().count() - counts['successful_jobs'] = models.UnifiedJob.objects.filter(status='successful').count() - counts['cancelled_jobs'] = models.UnifiedJob.objects.filter(status='canceled').count() - counts['failed_jobs'] = models.UnifiedJob.objects.filter(status='failed').count() - counts['error_jobs'] = models.UnifiedJob.objects.filter(status='error').count() # Do we also want to include error, new, pending, waiting, running? - counts['new_jobs'] = models.UnifiedJob.objects.filter(status='new').count() # Also, how much of this do we want `per instance` - counts['pending_jobs'] = models.UnifiedJob.objects.filter(status='pending').count() - counts['waiting_jobs'] = models.UnifiedJob.objects.filter(status='waiting').count() - counts['running_jobs'] = models.UnifiedJob.objects.filter(status='running').count() - - - # These will later be used to optimize the jobs_running and jobs_total python properties ^^ - # jobs_running = models.UnifiedJob.objects.filter(execution_node=instance, status__in=('running', 'waiting',)).count() - # jobs_total = models.UnifiedJob.objects.filter(execution_node=instance).count() - - return counts - - -@register('job_counts_instance') -def job_counts_instance(since): - counts = {} + counts['status'] = dict(models.UnifiedJob.objects.values_list('status').annotate(Count('status'))) for instance in models.Instance.objects.all(): counts[instance.id] = {'uuid': instance.uuid, - 'jobs_total': instance.jobs_total, # this is _all_ jobs run by that node - 'jobs_running': instance.jobs_running, # this is jobs in running & waiting state - 'launch_type': {'manual': models.UnifiedJob.objects.filter(launch_type='manual').count(), # I can definitely condense this - 'relaunch': models.UnifiedJob.objects.filter(launch_type='relaunch').count(), - 'scheduled': models.UnifiedJob.objects.filter(launch_type='scheduled').count(), - 'callback': models.UnifiedJob.objects.filter(launch_type='callback').count(), - 'dependency': models.UnifiedJob.objects.filter(launch_type='dependency').count(), - 'sync': models.UnifiedJob.objects.filter(launch_type='workflow').count(), - 'scm': models.UnifiedJob.objects.filter(launch_type='scm').count() - } - } + 'jobs_total': models.UnifiedJob.objects.filter(execution_node=instance.hostname, status__in=('running', 'waiting',)).count(), + 'jobs_running': models.UnifiedJob.objects.filter(execution_node=instance.hostname).count(), # jobs in running & waiting state + 'launch_type': dict(models.UnifiedJob.objects.filter(execution_node=instance.hostname).values_list('launch_type').annotate(Count('launch_type'))) + } return counts + @register('jobs') def jobs(since): counts = {} + jobs = models.Job.objects.filter(created__gt=since) counts['latest_jobs'] = models.Job.objects.filter(created__gt=since).count() return counts diff --git a/awx/main/analytics/core.py b/awx/main/analytics/core.py index 8b4c7ed38c..cdde5d56f2 100644 --- a/awx/main/analytics/core.py +++ b/awx/main/analytics/core.py @@ -31,7 +31,7 @@ def _valid_license(): logger.exception("A valid license was not found:") return False return True - + def register(key): """ @@ -73,6 +73,10 @@ def gather(dest=None, module=None): if last_run < max_interval or not last_run: last_run = max_interval + if settings.INSIGHTS_DATA_ENABLED: + logger.exception("Insights not enabled. Analytics data not gathered.") + return + if _valid_license() is False: logger.exception("Invalid License provided, or No License Provided") return diff --git a/awx/main/conf.py b/awx/main/conf.py index 06ab279823..4852f737c2 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -301,6 +301,16 @@ register( placeholder={'HTTP_PROXY': 'myproxy.local:8080'}, ) +register( + 'INSIGHTS_DATA_ENABLED', + field_class=fields.BooleanField, + default=False, + label=_('Insights Analytics Data'), + help_text=_('Enables Tower to gather analytics data about itself and send it to Insights.'), + category=_('Jobs'), + category_slug='jobs', +) + register( 'AWX_ROLES_ENABLED', field_class=fields.BooleanField, diff --git a/awx/main/migrations/0063_v350_analytics_state.py b/awx/main/migrations/0064_v350_analytics_state.py similarity index 92% rename from awx/main/migrations/0063_v350_analytics_state.py rename to awx/main/migrations/0064_v350_analytics_state.py index 9c02691b11..e01768694a 100644 --- a/awx/main/migrations/0063_v350_analytics_state.py +++ b/awx/main/migrations/0064_v350_analytics_state.py @@ -8,7 +8,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('main', '0062_v350_new_playbook_stats'), + ('main', '0063_v350_org_host_limits'), ] operations = [ diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index b1d2dc5726..c8dc520095 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -672,6 +672,12 @@ AWX_AUTO_DEPROVISION_INSTANCES = False # Note: This setting may be overridden by database settings. PENDO_TRACKING_STATE = "off" +# Enables Insights data collection for Ansible Tower. Defaults to the value of PENDO_TRACKING_STATE +if PENDO_TRACKING_STATE == "off": + INSIGHTS_DATA_ENABLED = False +else: + INSIGHTS_DATA_ENABLED = True + # Default list of modules allowed for ad hoc commands. # Note: This setting may be overridden by database settings. AD_HOC_COMMANDS = [ diff --git a/awx/settings/development.py b/awx/settings/development.py index 1b32e54e20..21a7952251 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -88,6 +88,7 @@ AWX_ISOLATED_LAUNCH_TIMEOUT = 30 # Disable Pendo on the UI for development/test. # Note: This setting may be overridden by database settings. PENDO_TRACKING_STATE = "off" +INSIGHTS_DATA_ENABLED = False # Use Django-Jenkins if installed. Only run tests for awx.main app. try: