From 78c0d531bcf73e7584caf3535950943699b6a511 Mon Sep 17 00:00:00 2001 From: Christian Adams Date: Thu, 22 Aug 2019 13:37:01 -0400 Subject: [PATCH] Adds versions to analytics collectors and manifest file. - adds 'query_info.json' to contain collection metadata - adds 'manifest.json' to contain collection file version info --- awx/main/analytics/__init__.py | 2 +- awx/main/analytics/collectors.py | 35 +++++++++++------ awx/main/analytics/core.py | 39 ++++++++++++++++--- .../management/commands/gather_analytics.py | 2 +- .../tests/functional/analytics/test_core.py | 6 +-- 5 files changed, 62 insertions(+), 22 deletions(-) diff --git a/awx/main/analytics/__init__.py b/awx/main/analytics/__init__.py index 0834f46157..9ab526f29b 100644 --- a/awx/main/analytics/__init__.py +++ b/awx/main/analytics/__init__.py @@ -1 +1 @@ -from .core import register, gather, ship # noqa +from .core import register, gather, ship, table_version # noqa diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index 0e0062aabc..a7dbbad287 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -12,14 +12,14 @@ from awx.main.utils import (get_awx_version, get_ansible_version, get_custom_venv_choices, camelcase_to_underscore) from awx.main import models from django.contrib.sessions.models import Session -from awx.main.analytics import register +from awx.main.analytics import register, table_version ''' This module is used to define metrics collected by awx.main.analytics.gather() Each function is decorated with a key name, and should return a data structure that can be serialized to JSON -@register('something') +@register('something', '1.0') def something(since): # the generated archive will contain a `something.json` w/ this JSON return {'some': 'json'} @@ -31,7 +31,7 @@ data _since_ the last report date - i.e., new data in the last 24 hours) ''' -@register('config') +@register('config', '1.0') def config(since): license_info = get_license(show_key=False) install_type = 'traditional' @@ -62,7 +62,7 @@ def config(since): } -@register('counts') +@register('counts', '1.0') def counts(since): counts = {} for cls in (models.Organization, models.Team, models.User, @@ -97,7 +97,7 @@ def counts(since): return counts -@register('org_counts') +@register('org_counts', '1.0') def org_counts(since): counts = {} for org in models.Organization.objects.annotate(num_users=Count('member_role__members', distinct=True), @@ -109,7 +109,7 @@ def org_counts(since): return counts -@register('cred_type_counts') +@register('cred_type_counts', '1.0') def cred_type_counts(since): counts = {} for cred_type in models.CredentialType.objects.annotate(num_credentials=Count( @@ -121,7 +121,7 @@ def cred_type_counts(since): return counts -@register('inventory_counts') +@register('inventory_counts', '1.0') def inventory_counts(since): counts = {} for inv in models.Inventory.objects.filter(kind='').annotate(num_sources=Count('inventory_sources', distinct=True), @@ -141,7 +141,7 @@ def inventory_counts(since): return counts -@register('projects_by_scm_type') +@register('projects_by_scm_type', '1.0') def projects_by_scm_type(since): counts = dict( (t[0] or 'manual', 0) @@ -160,7 +160,7 @@ def _get_isolated_datetime(last_check): return last_check -@register('instance_info') +@register('instance_info', '1.0') def instance_info(since, include_hostnames=False): info = {} instances = models.Instance.objects.values_list('hostname').values( @@ -182,7 +182,7 @@ def instance_info(since, include_hostnames=False): return info -@register('job_counts') +@register('job_counts', '1.0') def job_counts(since): counts = {} counts['total_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').count() @@ -192,7 +192,7 @@ def job_counts(since): return counts -@register('job_instance_counts') +@register('job_instance_counts', '1.0') def job_instance_counts(since): counts = {} job_types = models.UnifiedJob.objects.exclude(launch_type='sync').values_list( @@ -207,7 +207,19 @@ def job_instance_counts(since): return counts +@register('query_info', '1.0') +def query_info(since, collection_type): + query_info = {} + query_info['last_run'] = str(since) + query_info['current_time'] = str(now()) + query_info['collection_type'] = collection_type + return query_info + + # Copies Job Events from db to a .csv to be shipped +@table_version('events_table.csv', '1.0') +@table_version('unified_jobs_table.csv', '1.0') +@table_version('unified_job_template_table.csv', '1.0') def copy_tables(since, full_path): def _copy_table(table, query, path): file_path = os.path.join(path, table + '_table.csv') @@ -282,4 +294,3 @@ def copy_tables(since, full_path): ORDER BY main_unifiedjobtemplate.id ASC) TO STDOUT WITH CSV HEADER'''.format(since.strftime("'%Y-%m-%d %H:%M:%S'")) _copy_table(table='unified_job_template', query=unified_job_template_query, path=full_path) return - diff --git a/awx/main/analytics/core.py b/awx/main/analytics/core.py index af0ab8a05d..81a4814bdd 100644 --- a/awx/main/analytics/core.py +++ b/awx/main/analytics/core.py @@ -18,11 +18,13 @@ from awx.main.access import access_registry from awx.main.models.ha import TowerAnalyticsState -__all__ = ['register', 'gather', 'ship'] +__all__ = ['register', 'gather', 'ship', 'table_version'] logger = logging.getLogger('awx.main.analytics') +manifest = dict() + def _valid_license(): try: @@ -35,25 +37,37 @@ def _valid_license(): return True -def register(key): +def register(key, version): """ A decorator used to register a function as a metric collector. Decorated functions should return JSON-serializable objects. - @register('projects_by_scm_type') + @register('projects_by_scm_type', 1) def projects_by_scm_type(): return {'git': 5, 'svn': 1, 'hg': 0} """ def decorate(f): f.__awx_analytics_key__ = key + f.__awx_analytics_version__ = version return f return decorate -def gather(dest=None, module=None): +def table_version(file_name, version): + + global manifest + manifest[file_name] = version + + def decorate(f): + return f + + return decorate + + +def gather(dest=None, module=None, collection_type='scheduled'): """ Gather all defined metrics and write them as JSON files in a .tgz @@ -84,18 +98,33 @@ def gather(dest=None, module=None): from awx.main.analytics import collectors module = collectors + dest = dest or tempfile.mkdtemp(prefix='awx_analytics') for name, func in inspect.getmembers(module): if inspect.isfunction(func) and hasattr(func, '__awx_analytics_key__'): key = func.__awx_analytics_key__ + manifest['{}.json'.format(key)] = func.__awx_analytics_version__ path = '{}.json'.format(os.path.join(dest, key)) with open(path, 'w', encoding='utf-8') as f: try: - json.dump(func(last_run), f) + if func.__name__ == 'query_info': + json.dump(func(last_run, collection_type=collection_type), f) + else: + json.dump(func(last_run), f) except Exception: logger.exception("Could not generate metric {}.json".format(key)) f.close() os.remove(f.name) + + path = os.path.join(dest, 'manifest.json') + with open(path, 'w', encoding='utf-8') as f: + try: + json.dump(manifest, f) + except Exception: + logger.exception("Could not generate manifest.json") + f.close() + os.remove(f.name) + try: collectors.copy_tables(since=last_run, full_path=dest) except Exception: diff --git a/awx/main/management/commands/gather_analytics.py b/awx/main/management/commands/gather_analytics.py index 91708cb8c1..eac52d1ef3 100644 --- a/awx/main/management/commands/gather_analytics.py +++ b/awx/main/management/commands/gather_analytics.py @@ -23,7 +23,7 @@ class Command(BaseCommand): self.logger.propagate = False def handle(self, *args, **options): - tgz = gather() + tgz = gather(collection_type='manual') self.init_logging() if tgz: self.logger.debug(tgz) diff --git a/awx/main/tests/functional/analytics/test_core.py b/awx/main/tests/functional/analytics/test_core.py index f3996f6081..01f3858661 100644 --- a/awx/main/tests/functional/analytics/test_core.py +++ b/awx/main/tests/functional/analytics/test_core.py @@ -9,17 +9,17 @@ from django.conf import settings from awx.main.analytics import gather, register -@register('example') +@register('example', '1.0') def example(since): return {'awx': 123} -@register('bad_json') +@register('bad_json', '1.0') def bad_json(since): return set() -@register('throws_error') +@register('throws_error', '1.0') def throws_error(since): raise ValueError()