External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-04 02:01:01 -03:30
Code Issues Packages Projects Releases Wiki Activity

add basic job & JT analytics

Browse Source
This commit is contained in:
Christian Adams
2019-02-26 15:57:25 -05:00
parent c586fa9821
commit ae9377e0e4
4 changed files with 71 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
awx/main/analytics/collectors.py
View File

@@ -12,7 +12,6 @@ from django.contrib.sessions.models import Session
from awx.main.analytics import register from awx.main.analytics import register
# #
# This module is used to define metrics collected by awx.main.analytics.gather() # This module is used to define metrics collected by awx.main.analytics.gather()
# Each function is decorated with a key name, and should return a data # Each function is decorated with a key name, and should return a data
@@ -29,7 +28,6 @@ from awx.main.analytics import register
# data _since_ the last report date - i.e., new data in the last 24 hours) # data _since_ the last report date - i.e., new data in the last 24 hours)
# #
@register('config') @register('config')
def config(since): def config(since):
license_info = get_license(show_key=False) license_info = get_license(show_key=False)
@@ -45,6 +43,7 @@ def config(since):
'logging_aggregators': settings.LOG_AGGREGATOR_LOGGERS 'logging_aggregators': settings.LOG_AGGREGATOR_LOGGERS
} }
@register('counts') @register('counts')
def counts(since): def counts(since):
counts = {} counts = {}
@@ -78,20 +77,23 @@ def counts(since):
@register('org_counts') @register('org_counts')
def org_counts(since): def org_counts(since):
counts = {} counts = {}
for org in models.Organization.objects.annotate(num_users=Count('member_role__members', distinct=True), num_teams=Count('teams', distinct=True)): # Use .values to make a dict of only the fields we can about where for org in models.Organization.objects.annotate(
num_users=Count('member_role__members', distinct=True),
num_teams=Count('teams', distinct=True)): # use .only()
counts[org.id] = {'name': org.name, counts[org.id] = {'name': org.name,
'users': org.num_users, 'users': org.num_users,
'teams': org.num_teams 'teams': org.num_teams
} }
return counts return counts
@register('cred_type_counts') @register('cred_type_counts')
def cred_type_counts(since): def cred_type_counts(since):
counts = {} counts = {}
for cred_type in models.CredentialType.objects.annotate(num_credentials=Count('credentials', distinct=True)): for cred_type in models.CredentialType.objects.annotate(
num_credentials=Count('credentials', distinct=True)):
counts[cred_type.id] = {'name': cred_type.name, counts[cred_type.id] = {'name': cred_type.name,
'credential_count': cred_type.num_credentials 'credential_count': cred_type.num_credentials
} }
return counts return counts
@@ -100,12 +102,14 @@ def cred_type_counts(since):
def inventory_counts(since): def inventory_counts(since):
counts = {} counts = {}
from django.db.models import Count from django.db.models import Count
for inv in models.Inventory.objects.annotate(num_sources=Count('inventory_sources', distinct=True), num_hosts=Count('hosts', distinct=True)).only('id', 'name', 'kind'): for inv in models.Inventory.objects.annotate(
num_sources=Count('inventory_sources', distinct=True),
num_hosts=Count('hosts', distinct=True)).only('id', 'name', 'kind'):
counts[inv.id] = {'name': inv.name, counts[inv.id] = {'name': inv.name,
'kind': inv.kind, 'kind': inv.kind,
'hosts': inv.num_hosts, 'hosts': inv.num_hosts,
'sources': inv.num_sources 'sources': inv.num_sources
} }
return counts return counts
@@ -122,37 +126,48 @@ def projects_by_scm_type(since):
return counts return counts
@register('job_counts') #TODO: evaluate if we want this (was not an ask) Also, may think about annotating rather than grabbing objects for efficiency (even though there will likely be < 100 instances) @register('job_counts') # TODO: evaluate if we want this (was not an ask) Also, may think about annotating rather than grabbing objects for efficiency
def job_counts(since): def job_counts(since): # TODO: Optimize -- for example, all of these are going to need to be restrained to the last 24 hours/INSIGHTS_SCHEDULE
counts = {} counts = {}
counts['total_jobs'] = models.UnifiedJob.objects.all().count() counts['total_jobs'] = models.UnifiedJob.objects.all().count()
for instance in models.Instance.objects.all(): counts['successful_jobs'] = models.UnifiedJob.objects.filter(status='successful').count()
counts[instance.id] = {'uuid': instance.uuid, counts['cancelled_jobs'] = models.UnifiedJob.objects.filter(status='canceled').count()
'jobs_total': instance.jobs_total, # this is _all_ jobs run by that node counts['failed_jobs'] = models.UnifiedJob.objects.filter(status='failed').count()
'jobs_running': instance.jobs_running, # this is jobs in running & waiting state counts['error_jobs'] = models.UnifiedJob.objects.filter(status='error').count() # Do we also want to include error, new, pending, waiting, running?
} counts['new_jobs'] = models.UnifiedJob.objects.filter(status='new').count() # Also, how much of this do we want `per instance`
jobs_running = models.UnifiedJob.objects.filter(execution_node=instance, status__in=('running', 'waiting',)).count() counts['pending_jobs'] = models.UnifiedJob.objects.filter(status='pending').count()
jobs_total = models.UnifiedJob.objects.filter(execution_node=instance).count() counts['waiting_jobs'] = models.UnifiedJob.objects.filter(status='waiting').count()
counts['running_jobs'] = models.UnifiedJob.objects.filter(status='running').count()
# These will later be used to optimize the jobs_running and jobs_total python properties ^^
counts['total_jobs'] = models.UnifiedJob.objects.annotate(running_jobs=) # jobs_running = models.UnifiedJob.objects.filter(execution_node=instance, status__in=('running', 'waiting',)).count()
for instance in models.Instance.objects.all(): # jobs_total = models.UnifiedJob.objects.filter(execution_node=instance).count()
counts[instance.id] = {'uuid': instance.uuid,
'jobs_total': instance.jobs_total, # this is _all_ jobs run by that node
'jobs_running': instance.jobs_running, # this is jobs in running & waiting state
}
jobs_running = models.UnifiedJob.objects.filter(execution_node=instance, status__in=('running', 'waiting',)).count()
jobs_total = models.UnifiedJob.objects.filter(execution_node=instance).count()
return counts return counts
@register('job_counts_instance')
def job_counts_instance(since):
counts = {}
for instance in models.Instance.objects.all():
counts[instance.id] = {'uuid': instance.uuid,
'jobs_total': instance.jobs_total, # this is _all_ jobs run by that node
'jobs_running': instance.jobs_running, # this is jobs in running & waiting state
'launch_type': {'manual': models.UnifiedJob.objects.filter(launch_type='manual').count(), # I can definitely condense this
'relaunch': models.UnifiedJob.objects.filter(launch_type='relaunch').count(),
'scheduled': models.UnifiedJob.objects.filter(launch_type='scheduled').count(),
'callback': models.UnifiedJob.objects.filter(launch_type='callback').count(),
'dependency': models.UnifiedJob.objects.filter(launch_type='dependency').count(),
'sync': models.UnifiedJob.objects.filter(launch_type='workflow').count(),
'scm': models.UnifiedJob.objects.filter(launch_type='scm').count()
}
}
return counts
@register('jobs') @register('jobs')
def jobs(since): def jobs(since):
counts = {} counts = {}
jobs = models.Job.objects.filter(created__gt=since)
counts['latest_jobs'] = models.Job.objects.filter(created__gt=since).count() counts['latest_jobs'] = models.Job.objects.filter(created__gt=since).count()
return counts return counts

2
awx/main/migrations/0062_v350_analytics_state.py → awx/main/migrations/0063_v350_analytics_state.py
View File

@@ -8,7 +8,7 @@ from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('main', '0061_v350_track_native_credentialtype_source'), ('main', '0062_v350_new_playbook_stats'),
] ]
operations = [ operations = [

1
awx/main/tasks.py
View File

@@ -70,6 +70,7 @@ from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja
from awx.main.utils.reload import stop_local_services from awx.main.utils.reload import stop_local_services
from awx.main.utils.pglock import advisory_lock from awx.main.utils.pglock import advisory_lock
from awx.main.consumers import emit_channel_notification from awx.main.consumers import emit_channel_notification
from awx.main import analytics
from awx.conf import settings_registry from awx.conf import settings_registry
from rest_framework.exceptions import PermissionDenied from rest_framework.exceptions import PermissionDenied

32
awx/main/tests/functional/analytics/test_counts.py
View File

@@ -7,18 +7,26 @@ from awx.main.analytics import collectors
@pytest.mark.django_db @pytest.mark.django_db
def test_empty(): def test_empty():
assert collectors.counts(None) == { assert collectors.counts(None) == {
'organization': 0, "active_api_sessions": 0,
'team': 0, "active_channels_sessions": 0,
'user': 0, "active_host_count": 0,
'inventory': 0, "active_sessions": 0,
'credential': 0, "credential": 0,
'project': 0, "custom_inventory_script": 0,
'job_template': 0, "custom_virtualenvs": 1, # dev env ansible3
'workflow_job_template': 0, "host": 0,
'host': 0, "inventory": 0,
'schedule': 0, "job_template": 0,
'custom_inventory_script': 0, "normal_inventories": 0,
'custom_virtualenvs': 1 # dev env ansible3 "notification_template": 0,
"organization": 0,
"project": 0,
"running_jobs": 0,
"schedule": 0,
"smart_inventories": 0,
"team": 0,
"user": 0,
"workflow_job_template": 0
} }