add a minimal framework for generating analytics/metrics

annotate queries & add license analytics

awx/main/analytics/__init__.py

@@ -0,0 +1 @@
+from .core import register, gather, ship  # noqa

awx/main/analytics/collectors.py

@@ -0,0 +1,158 @@
+import os.path
+
+from django.db.models import Count
+from django.conf import settings
+from django.utils.timezone import now
+
+from awx.conf.license import get_license
+from awx.main.utils import (get_awx_version, get_ansible_version,
+                            get_custom_venv_choices, camelcase_to_underscore)
+from awx.main import models
+from django.contrib.sessions.models import Session
+from awx.main.analytics import register
+
+
+
+#
+# This module is used to define metrics collected by awx.main.analytics.gather()
+# Each function is decorated with a key name, and should return a data
+# structure that can be serialized to JSON
+#
+# @register('something')
+# def something(since):
+#     # the generated archive will contain a `something.json` w/ this JSON
+#     return {'some': 'json'}
+#
+# All functions - when called - will be passed a datetime.datetime object,
+# `since`, which represents the last time analytics were gathered (some metrics
+# functions - like those that return metadata about playbook runs, may return
+# data _since_ the last report date - i.e., new data in the last 24 hours)
+#
+
+
+@register('config')
+def config(since):
+    license_info = get_license(show_key=False)
+    return {
+        'system_uuid': settings.SYSTEM_UUID,
+        'tower_url_base': settings.TOWER_URL_BASE,
+        'tower_version': get_awx_version(),
+        'ansible_version': get_ansible_version(),
+        'license_type': license_info.get('license_type', 'UNLICENSED'),
+        'free_instances': license_info.get('free instances', 0),
+        'license_expiry': license_info.get('time_remaining', 0),
+        'authentication_backends': settings.AUTHENTICATION_BACKENDS,
+        'logging_aggregators': settings.LOG_AGGREGATOR_LOGGERS
+    }
+
+@register('counts')
+def counts(since):
+    counts = {}
+    for cls in (models.Organization, models.Team, models.User,
+                models.Inventory, models.Credential, models.Project,
+                models.JobTemplate, models.WorkflowJobTemplate, models.Host,
+                models.Schedule, models.CustomInventoryScript,
+                models.NotificationTemplate):
+        counts[camelcase_to_underscore(cls.__name__)] = cls.objects.count()
+
+    venvs = get_custom_venv_choices()
+    counts['custom_virtualenvs'] = len([
+        v for v in venvs
+        if os.path.basename(v.rstrip('/')) != 'ansible'
+    ])
+
+    counts['active_host_count'] = models.Host.objects.active_count()
+    counts['smart_inventories'] = models.Inventory.objects.filter(kind='smart').count()
+    counts['normal_inventories'] = models.Inventory.objects.filter(kind='').count()
+
+    active_sessions = Session.objects.filter(expire_date__gte=now()).count()
+    api_sessions = models.UserSessionMembership.objects.select_related('session').filter(session__expire_date__gte=now()).count()
+    channels_sessions = active_sessions - api_sessions
+    counts['active_sessions'] = active_sessions
+    counts['active_api_sessions'] = api_sessions
+    counts['active_channels_sessions'] = channels_sessions
+    counts['running_jobs'] = models.Job.objects.filter(status='running').count()
+    return counts
+    
+    
+@register('org_counts')
+def org_counts(since):
+    counts = {}
+    for org in models.Organization.objects.annotate(num_users=Count('member_role__members', distinct=True), num_teams=Count('teams', distinct=True)):  # Use .values to make a dict of only the fields we can about where
+        counts[org.id] = {'name': org.name,
+                            'users': org.num_users,
+                            'teams': org.num_teams
+                            }
+    return counts
+    
+    
+@register('cred_type_counts')
+def cred_type_counts(since):
+    counts = {}
+    for cred_type in models.CredentialType.objects.annotate(num_credentials=Count('credentials', distinct=True)):  
+        counts[cred_type.id] = {'name': cred_type.name,
+                                  'credential_count': cred_type.num_credentials
+                                }
+    return counts
+    
+    
+@register('inventory_counts')
+def inventory_counts(since):
+    counts = {}
+    from django.db.models import Count
+    for inv in models.Inventory.objects.annotate(num_sources=Count('inventory_sources', distinct=True), num_hosts=Count('hosts', distinct=True)).only('id', 'name', 'kind'):
+        counts[inv.id] = {'name': inv.name,
+                            'kind': inv.kind,
+                            'hosts': inv.num_hosts,
+                            'sources': inv.num_sources
+                            }
+    return counts
+
+
+@register('projects_by_scm_type')
+def projects_by_scm_type(since):
+    counts = dict(
+        (t[0] or 'manual', 0)
+        for t in models.Project.SCM_TYPE_CHOICES
+    )
+    for result in models.Project.objects.values('scm_type').annotate(
+        count=Count('scm_type')
+    ).order_by('scm_type'):
+        counts[result['scm_type'] or 'manual'] = result['count']
+    return counts
+
+
+@register('job_counts')   #TODO: evaluate if we want this (was not an ask)  Also, may think about annotating rather than grabbing objects for efficiency (even though there will likely be < 100 instances)
+def job_counts(since):
+    counts = {}
+    
+
+    counts['total_jobs'] = models.UnifiedJob.objects.all().count()
+    for instance in models.Instance.objects.all():
+        counts[instance.id] = {'uuid': instance.uuid,
+                                'jobs_total': instance.jobs_total,       # this is _all_ jobs run by that node
+                                'jobs_running': instance.jobs_running,   # this is jobs in running & waiting state
+                                }
+        jobs_running = models.UnifiedJob.objects.filter(execution_node=instance, status__in=('running', 'waiting',)).count()
+        jobs_total = models.UnifiedJob.objects.filter(execution_node=instance).count()
+        
+        
+    counts['total_jobs'] = models.UnifiedJob.objects.annotate(running_jobs=)
+    for instance in models.Instance.objects.all():
+        counts[instance.id] = {'uuid': instance.uuid,
+                                'jobs_total': instance.jobs_total,       # this is _all_ jobs run by that node
+                                'jobs_running': instance.jobs_running,   # this is jobs in running & waiting state
+                                }
+        jobs_running = models.UnifiedJob.objects.filter(execution_node=instance, status__in=('running', 'waiting',)).count()
+        jobs_total = models.UnifiedJob.objects.filter(execution_node=instance).count()
+        
+    return counts
+    
+    
+    
+@register('jobs')
+def jobs(since):
+    counts = {}
+    jobs = models.Job.objects.filter(created__gt=since)
+    counts['latest_jobs'] = models.Job.objects.filter(created__gt=since).count()
+    return counts

awx/main/analytics/core.py

@@ -0,0 +1,130 @@
+import codecs
+import inspect
+import json
+import logging
+import os
+import os.path
+import tempfile
+import shutil
+import subprocess
+
+from django.conf import settings
+from django.utils.encoding import smart_str
+from django.utils.timezone import now, timedelta
+from rest_framework.exceptions import PermissionDenied
+
+from awx.main.models import Job
+from awx.main.access import access_registry
+from awx.main.models.ha import TowerAnalyticsState
+
+
+__all__ = ['register', 'gather', 'ship']
+
+
+logger = logging.getLogger('awx.main.analytics')
+
+
+def _valid_license():
+    try:
+        access_registry[Job](None).check_license()
+    except PermissionDenied:
+        logger.exception("A valid license was not found:")
+        return False
+    return True
+
+
+def register(key):
+    """
+    A decorator used to register a function as a metric collector.
+
+    Decorated functions should return JSON-serializable objects.
+
+    @register('projects_by_scm_type')
+    def projects_by_scm_type():
+        return {'git': 5, 'svn': 1, 'hg': 0}
+    """
+
+    def decorate(f):
+        f.__awx_analytics_key__ = key
+        return f
+
+    return decorate
+
+
+def gather(dest=None, module=None):
+    """
+    Gather all defined metrics and write them as JSON files in a .tgz
+
+    :param dest:    the (optional) absolute path to write a compressed tarball
+    :pararm module: the module to search for registered analytic collector
+                    functions; defaults to awx.main.analytics.collectors
+    """
+    import time                 # TODO: Remove this
+    start_time = time.time()    # TODO: Remove this
+    
+    run_now = now()
+    state = TowerAnalyticsState.get_solo()
+    last_run = state.last_run
+    logger.debug("Last analytics run was: {}".format(last_run))
+    state.last_run = run_now
+    state.save()
+    
+    max_interval = now() - timedelta(days=7)
+    if last_run < max_interval or not last_run:
+        last_run = max_interval
+
+    if _valid_license() is False:
+        logger.exception("Invalid License provided, or No License Provided")
+        return
+
+    if module is None:
+        from awx.main.analytics import collectors
+        module = collectors
+
+    dest = dest or tempfile.mkdtemp(prefix='awx_analytics')
+    for name, func in inspect.getmembers(module):
+        if inspect.isfunction(func) and hasattr(func, '__awx_analytics_key__'):
+            key = func.__awx_analytics_key__
+            path = '{}.json'.format(os.path.join(dest, key))
+            with codecs.open(path, 'w', encoding='utf-8') as f:
+                try:
+                    json.dump(func(last_run), f)
+                except Exception:
+                    logger.exception("Could not generate metric {}.json".format(key))
+                    f.close()
+                    os.remove(f.name)
+
+    # can't use isoformat() since it has colons, which GNU tar doesn't like
+    tarname = '_'.join([
+        settings.SYSTEM_UUID,
+        run_now.strftime('%Y-%m-%d-%H%M%S%z')
+    ])
+    tgz = shutil.make_archive(
+        os.path.join(os.path.dirname(dest), tarname),
+        'gztar',
+        dest
+    )
+    shutil.rmtree(dest)
+    print("Analytics Time --- %s seconds ---" % (time.time() - start_time))  # TODO: Remove this
+    return tgz
+
+
+def ship(path):
+    """
+    Ship gathered metrics via the Insights agent
+    """
+    agent = 'insights-client'
+    if shutil.which(agent) is None:
+        logger.error('could not find {} on PATH'.format(agent))
+        return
+    logger.debug('shipping analytics file: {}'.format(path))
+    try:
+        cmd = [
+            agent, '--payload', path, '--content-type', settings.INSIGHTS_AGENT_MIME
+        ]
+        output = smart_str(subprocess.check_output(cmd, timeout=60 * 5))
+        logger.debug(output)
+    except subprocess.CalledProcessError:
+        logger.exception('{} failure:'.format(cmd))
+    except subprocess.TimeoutExpired:
+        logger.exception('{} timeout:'.format(cmd))