diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index c5c08d72ca..b0462221e0 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -12,7 +12,6 @@ from django.contrib.sessions.models import Session from awx.main.analytics import register - # # This module is used to define metrics collected by awx.main.analytics.gather() # Each function is decorated with a key name, and should return a data @@ -29,7 +28,6 @@ from awx.main.analytics import register # data _since_ the last report date - i.e., new data in the last 24 hours) # - @register('config') def config(since): license_info = get_license(show_key=False) @@ -45,6 +43,7 @@ def config(since): 'logging_aggregators': settings.LOG_AGGREGATOR_LOGGERS } + @register('counts') def counts(since): counts = {} @@ -78,20 +77,23 @@ def counts(since): @register('org_counts') def org_counts(since): counts = {} - for org in models.Organization.objects.annotate(num_users=Count('member_role__members', distinct=True), num_teams=Count('teams', distinct=True)): # Use .values to make a dict of only the fields we can about where + for org in models.Organization.objects.annotate( + num_users=Count('member_role__members', distinct=True), + num_teams=Count('teams', distinct=True)): # use .only() counts[org.id] = {'name': org.name, - 'users': org.num_users, - 'teams': org.num_teams - } + 'users': org.num_users, + 'teams': org.num_teams + } return counts @register('cred_type_counts') def cred_type_counts(since): counts = {} - for cred_type in models.CredentialType.objects.annotate(num_credentials=Count('credentials', distinct=True)): + for cred_type in models.CredentialType.objects.annotate( + num_credentials=Count('credentials', distinct=True)): counts[cred_type.id] = {'name': cred_type.name, - 'credential_count': cred_type.num_credentials + 'credential_count': cred_type.num_credentials } return counts @@ -100,12 +102,14 @@ def cred_type_counts(since): def inventory_counts(since): counts = {} from django.db.models import Count - for inv in models.Inventory.objects.annotate(num_sources=Count('inventory_sources', distinct=True), num_hosts=Count('hosts', distinct=True)).only('id', 'name', 'kind'): + for inv in models.Inventory.objects.annotate( + num_sources=Count('inventory_sources', distinct=True), + num_hosts=Count('hosts', distinct=True)).only('id', 'name', 'kind'): counts[inv.id] = {'name': inv.name, - 'kind': inv.kind, - 'hosts': inv.num_hosts, - 'sources': inv.num_sources - } + 'kind': inv.kind, + 'hosts': inv.num_hosts, + 'sources': inv.num_sources + } return counts @@ -122,37 +126,48 @@ def projects_by_scm_type(since): return counts -@register('job_counts') #TODO: evaluate if we want this (was not an ask) Also, may think about annotating rather than grabbing objects for efficiency (even though there will likely be < 100 instances) -def job_counts(since): +@register('job_counts') # TODO: evaluate if we want this (was not an ask) Also, may think about annotating rather than grabbing objects for efficiency +def job_counts(since): # TODO: Optimize -- for example, all of these are going to need to be restrained to the last 24 hours/INSIGHTS_SCHEDULE counts = {} - - counts['total_jobs'] = models.UnifiedJob.objects.all().count() - for instance in models.Instance.objects.all(): - counts[instance.id] = {'uuid': instance.uuid, - 'jobs_total': instance.jobs_total, # this is _all_ jobs run by that node - 'jobs_running': instance.jobs_running, # this is jobs in running & waiting state - } - jobs_running = models.UnifiedJob.objects.filter(execution_node=instance, status__in=('running', 'waiting',)).count() - jobs_total = models.UnifiedJob.objects.filter(execution_node=instance).count() + counts['successful_jobs'] = models.UnifiedJob.objects.filter(status='successful').count() + counts['cancelled_jobs'] = models.UnifiedJob.objects.filter(status='canceled').count() + counts['failed_jobs'] = models.UnifiedJob.objects.filter(status='failed').count() + counts['error_jobs'] = models.UnifiedJob.objects.filter(status='error').count() # Do we also want to include error, new, pending, waiting, running? + counts['new_jobs'] = models.UnifiedJob.objects.filter(status='new').count() # Also, how much of this do we want `per instance` + counts['pending_jobs'] = models.UnifiedJob.objects.filter(status='pending').count() + counts['waiting_jobs'] = models.UnifiedJob.objects.filter(status='waiting').count() + counts['running_jobs'] = models.UnifiedJob.objects.filter(status='running').count() + - - counts['total_jobs'] = models.UnifiedJob.objects.annotate(running_jobs=) - for instance in models.Instance.objects.all(): - counts[instance.id] = {'uuid': instance.uuid, - 'jobs_total': instance.jobs_total, # this is _all_ jobs run by that node - 'jobs_running': instance.jobs_running, # this is jobs in running & waiting state - } - jobs_running = models.UnifiedJob.objects.filter(execution_node=instance, status__in=('running', 'waiting',)).count() - jobs_total = models.UnifiedJob.objects.filter(execution_node=instance).count() + # These will later be used to optimize the jobs_running and jobs_total python properties ^^ + # jobs_running = models.UnifiedJob.objects.filter(execution_node=instance, status__in=('running', 'waiting',)).count() + # jobs_total = models.UnifiedJob.objects.filter(execution_node=instance).count() return counts +@register('job_counts_instance') +def job_counts_instance(since): + counts = {} + for instance in models.Instance.objects.all(): + counts[instance.id] = {'uuid': instance.uuid, + 'jobs_total': instance.jobs_total, # this is _all_ jobs run by that node + 'jobs_running': instance.jobs_running, # this is jobs in running & waiting state + 'launch_type': {'manual': models.UnifiedJob.objects.filter(launch_type='manual').count(), # I can definitely condense this + 'relaunch': models.UnifiedJob.objects.filter(launch_type='relaunch').count(), + 'scheduled': models.UnifiedJob.objects.filter(launch_type='scheduled').count(), + 'callback': models.UnifiedJob.objects.filter(launch_type='callback').count(), + 'dependency': models.UnifiedJob.objects.filter(launch_type='dependency').count(), + 'sync': models.UnifiedJob.objects.filter(launch_type='workflow').count(), + 'scm': models.UnifiedJob.objects.filter(launch_type='scm').count() + } + } + return counts + @register('jobs') def jobs(since): counts = {} - jobs = models.Job.objects.filter(created__gt=since) counts['latest_jobs'] = models.Job.objects.filter(created__gt=since).count() return counts diff --git a/awx/main/migrations/0062_v350_analytics_state.py b/awx/main/migrations/0063_v350_analytics_state.py similarity index 90% rename from awx/main/migrations/0062_v350_analytics_state.py rename to awx/main/migrations/0063_v350_analytics_state.py index d852df3530..9c02691b11 100644 --- a/awx/main/migrations/0062_v350_analytics_state.py +++ b/awx/main/migrations/0063_v350_analytics_state.py @@ -8,7 +8,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('main', '0061_v350_track_native_credentialtype_source'), + ('main', '0062_v350_new_playbook_stats'), ] operations = [ diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 1f4992fbf9..077d4a971b 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -70,6 +70,7 @@ from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja from awx.main.utils.reload import stop_local_services from awx.main.utils.pglock import advisory_lock from awx.main.consumers import emit_channel_notification +from awx.main import analytics from awx.conf import settings_registry from rest_framework.exceptions import PermissionDenied diff --git a/awx/main/tests/functional/analytics/test_counts.py b/awx/main/tests/functional/analytics/test_counts.py index 1cb043c30d..36b68675d1 100644 --- a/awx/main/tests/functional/analytics/test_counts.py +++ b/awx/main/tests/functional/analytics/test_counts.py @@ -7,18 +7,26 @@ from awx.main.analytics import collectors @pytest.mark.django_db def test_empty(): assert collectors.counts(None) == { - 'organization': 0, - 'team': 0, - 'user': 0, - 'inventory': 0, - 'credential': 0, - 'project': 0, - 'job_template': 0, - 'workflow_job_template': 0, - 'host': 0, - 'schedule': 0, - 'custom_inventory_script': 0, - 'custom_virtualenvs': 1 # dev env ansible3 + "active_api_sessions": 0, + "active_channels_sessions": 0, + "active_host_count": 0, + "active_sessions": 0, + "credential": 0, + "custom_inventory_script": 0, + "custom_virtualenvs": 1, # dev env ansible3 + "host": 0, + "inventory": 0, + "job_template": 0, + "normal_inventories": 0, + "notification_template": 0, + "organization": 0, + "project": 0, + "running_jobs": 0, + "schedule": 0, + "smart_inventories": 0, + "team": 0, + "user": 0, + "workflow_job_template": 0 }