From c03cef022dbf44b0f32ac0694c61389641440138 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 3 Apr 2015 09:58:38 -0400 Subject: [PATCH 1/7] Implemented fact scan storage logic. * added mongo connection logic * added mongo dbtransform logic to allow keys with . and $ * altered tower fact scanner CacheModule to emit a message for each fact module facts (including ansible facts). Previously, seperate facts module facts were getting concatenated to each subsequent emi * tower fact scanner CacheModule timeout set as to not hang for forever * broke apart commands.py test * added unit test for run_fact_cache_receiver, facts, and dbtransform --- awx/main/dbtransform.py | 55 +++++ .../commands/run_fact_cache_receiver.py | 121 +++++------ awx/main/models/__init__.py | 1 + awx/main/models/fact.py | 49 +++++ awx/main/tests/__init__.py | 3 + awx/main/tests/base.py | 8 + awx/main/tests/commands/__init__.py | 5 + awx/main/tests/commands/base.py | 89 ++++++++ .../commands_monolithic.py} | 0 .../tests/commands/run_fact_cache_receiver.py | 199 ++++++++++++++++++ awx/main/tests/dbtransform.py | 78 +++++++ awx/main/tests/jobs/__init__.py | 3 + awx/main/tests/models/__init__.py | 4 + awx/main/tests/models/fact.py | 87 ++++++++ awx/plugins/fact_caching/tower.py | 52 ++++- awx/settings/__init__.py | 8 + awx/settings/defaults.py | 2 + awx/settings/development.py | 2 + requirements/dev_local.txt | 3 +- 19 files changed, 698 insertions(+), 71 deletions(-) create mode 100644 awx/main/dbtransform.py create mode 100644 awx/main/models/fact.py create mode 100644 awx/main/tests/commands/__init__.py create mode 100644 awx/main/tests/commands/base.py rename awx/main/tests/{commands.py => commands/commands_monolithic.py} (100%) create mode 100644 awx/main/tests/commands/run_fact_cache_receiver.py create mode 100644 awx/main/tests/dbtransform.py create mode 100644 awx/main/tests/models/__init__.py create mode 100644 awx/main/tests/models/fact.py diff --git a/awx/main/dbtransform.py b/awx/main/dbtransform.py new file mode 100644 index 0000000000..f88708a467 --- /dev/null +++ b/awx/main/dbtransform.py @@ -0,0 +1,55 @@ +# Copyright (c) 2014, Ansible, Inc. +# All Rights Reserved. +from pymongo.son_manipulator import SONManipulator + +''' +Inspired by: https://stackoverflow.com/questions/8429318/how-to-use-dot-in-field-name/20698802#20698802 + +Replace . and $ with unicode values +''' +class KeyTransform(SONManipulator): + def __init__(self, replace): + self.replace = replace + + def transform_key(self, key, replace, replacement): + """Transform key for saving to database.""" + return key.replace(replace, replacement) + + def revert_key(self, key, replace, replacement): + """Restore transformed key returning from database.""" + return key.replace(replacement, replace) + + def transform_incoming(self, son, collection): + """Recursively replace all keys that need transforming.""" + for (key, value) in son.items(): + for r in self.replace: + replace = r[0] + replacement = r[1] + if replace in key: + if isinstance(value, dict): + son[self.transform_key(key, replace, replacement)] = self.transform_incoming( + son.pop(key), collection) + else: + son[self.transform_key(key, replace, replacement)] = son.pop(key) + elif isinstance(value, dict): # recurse into sub-docs + son[key] = self.transform_incoming(value, collection) + return son + + def transform_outgoing(self, son, collection): + """Recursively restore all transformed keys.""" + for (key, value) in son.items(): + for r in self.replace: + replace = r[0] + replacement = r[1] + if replacement in key: + if isinstance(value, dict): + son[self.revert_key(key, replace, replacement)] = self.transform_outgoing( + son.pop(key), collection) + else: + son[self.revert_key(key, replace, replacement)] = son.pop(key) + elif isinstance(value, dict): # recurse into sub-docs + son[key] = self.transform_outgoing(value, collection) + return son + +def register_key_transform(db): + db.add_son_manipulator(KeyTransform([('.', '\uff0E'), ('$', '\uff04')])) diff --git a/awx/main/management/commands/run_fact_cache_receiver.py b/awx/main/management/commands/run_fact_cache_receiver.py index 827ce5adaa..f31a8495a1 100644 --- a/awx/main/management/commands/run_fact_cache_receiver.py +++ b/awx/main/management/commands/run_fact_cache_receiver.py @@ -2,88 +2,71 @@ # All Rights Reserved import logging +from datetime import datetime +import json from django.core.management.base import NoArgsCommand from awx.main.models import * # noqa from awx.main.socket import Socket +import pymongo from pymongo import MongoClient +_MODULES = [ 'packages', 'services', 'files' ] + logger = logging.getLogger('awx.main.commands.run_fact_cache_receiver') - -from pymongo.son_manipulator import SONManipulator - -class KeyTransform(SONManipulator): - """Transforms keys going to database and restores them coming out. - - This allows keys with dots in them to be used (but does break searching on - them unless the find command also uses the transform. - - Example & test: - # To allow `.` (dots) in keys - import pymongo - client = pymongo.MongoClient("mongodb://localhost") - db = client['delete_me'] - db.add_son_manipulator(KeyTransform(".", "_dot_")) - db['mycol'].remove() - db['mycol'].update({'_id': 1}, {'127.0.0.1': 'localhost'}, upsert=True, - manipulate=True) - print db['mycol'].find().next() - print db['mycol'].find({'127_dot_0_dot_0_dot_1': 'localhost'}).next() - - Note: transformation could be easily extended to be more complex. - """ - - def __init__(self, replace, replacement): - self.replace = replace - self.replacement = replacement - - def transform_key(self, key): - """Transform key for saving to database.""" - return key.replace(self.replace, self.replacement) - - def revert_key(self, key): - """Restore transformed key returning from database.""" - return key.replace(self.replacement, self.replace) - - def transform_incoming(self, son, collection): - """Recursively replace all keys that need transforming.""" - for (key, value) in son.items(): - if self.replace in key: - if isinstance(value, dict): - son[self.transform_key(key)] = self.transform_incoming( - son.pop(key), collection) - else: - son[self.transform_key(key)] = son.pop(key) - elif isinstance(value, dict): # recurse into sub-docs - son[key] = self.transform_incoming(value, collection) - return son - - def transform_outgoing(self, son, collection): - return son - class FactCacheReceiver(object): - def __init__(self): - self.client = MongoClient('localhost', 27017) - + self.timestamp = None + + def _determine_module(self, facts): + for x in _MODULES: + if x in facts: + return x + return 'ansible' + + def _extract_module_facts(self, module, facts): + if module in facts: + f = facts[module] + return f + return facts + + def process_facts(self, facts): + module = self._determine_module(facts) + facts = self._extract_module_facts(module, facts) + return (module, facts) + def process_fact_message(self, message): - host = message['host'].replace(".", "_") - facts = message['facts'] + hostname = message['host'] + facts_data = message['facts'] date_key = message['date_key'] - host_db = self.client.host_facts - host_db.add_son_manipulator(KeyTransform(".", "_")) - host_db.add_son_manipulator(KeyTransform("$", "_")) - host_collection = host_db[host] - facts.update(dict(tower_host=host, datetime=date_key)) - rec = host_collection.find({"datetime": date_key}) - if rec.count(): - this_fact = rec.next() - this_fact.update(facts) - host_collection.save(this_fact) - else: - host_collection.insert(facts) + + # TODO: in ansible < v2 module_setup is emitted for "smart" fact caching. + # ansible v2 will not emit this message. Thus, this can be removed at that time. + if 'module_setup' in facts_data and len(facts_data) == 1: + return + + try: + host = FactHost.objects.get(hostname=hostname) + except FactHost.DoesNotExist as e: + host = FactHost(hostname=hostname) + host.save() + except FactHost.MultipleObjectsReturned as e: + query = "db['fact_host'].find(hostname=%s)" % hostname + print('Database inconsistent. Multiple FactHost "%s" exist. Try the query %s to find the records.' % (hostname, query)) + return + + (module, facts) = self.process_facts(facts_data) + self.timestamp = datetime.fromtimestamp(date_key, None) + + try: + # Update existing Fact entry + version_obj = FactVersion.objects.get(timestamp=self.timestamp, host=host, module=module) + Fact.objects(id=version_obj.fact.id).update_one(fact=facts) + except FactVersion.DoesNotExist: + # Create new Fact entry + (fact_obj, version_obj) = Fact.add_fact(self.timestamp, facts, host, module) def run_receiver(self): with Socket('fact_cache', 'r') as facts: diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index b7d57212de..9d89df393e 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -16,6 +16,7 @@ from awx.main.models.ad_hoc_commands import * # noqa from awx.main.models.schedules import * # noqa from awx.main.models.activity_stream import * # noqa from awx.main.models.ha import * # noqa +from awx.main.models.fact import * # noqa # Monkeypatch Django serializer to ignore django-taggit fields (which break # the dumpdata command; see https://github.com/alex/django-taggit/issues/155). diff --git a/awx/main/models/fact.py b/awx/main/models/fact.py new file mode 100644 index 0000000000..b97b11d7e3 --- /dev/null +++ b/awx/main/models/fact.py @@ -0,0 +1,49 @@ +from mongoengine import Document, DynamicDocument, DateTimeField, ReferenceField, StringField + +class FactHost(Document): + hostname = StringField(max_length=100, required=True, unique=True) + + # TODO: Consider using hashed index on hostname. django-mongo may not support this but + # executing raw js will + meta = { + 'indexes': [ + 'hostname' + ] + } + +class Fact(DynamicDocument): + timestamp = DateTimeField(required=True) + host = ReferenceField(FactHost, required=True) + module = StringField(max_length=50, required=True) + # fact = + + # TODO: Consider using hashed index on host. django-mongo may not support this but + # executing raw js will + meta = { + 'indexes': [ + '-timestamp', + 'host' + ] + } + + @staticmethod + def add_fact(timestamp, fact, host, module): + fact_obj = Fact(timestamp=timestamp, host=host, module=module, fact=fact) + fact_obj.save() + version_obj = FactVersion(timestamp=timestamp, host=host, module=module, fact=fact_obj) + version_obj.save() + return (fact_obj, version_obj) + +class FactVersion(Document): + timestamp = DateTimeField(required=True) + host = ReferenceField(FactHost, required=True) + module = StringField(max_length=50, required=True) + fact = ReferenceField(Fact, required=True) + # TODO: Consider using hashed index on module. django-mongo may not support this but + # executing raw js will + meta = { + 'indexes': [ + '-timestamp', + 'module' + ] + } diff --git a/awx/main/tests/__init__.py b/awx/main/tests/__init__.py index 6ba54a7802..29d772d495 100644 --- a/awx/main/tests/__init__.py +++ b/awx/main/tests/__init__.py @@ -15,3 +15,6 @@ from awx.main.tests.activity_stream import * # noqa from awx.main.tests.schedules import * # noqa from awx.main.tests.redact import * # noqa from awx.main.tests.views import * # noqa +from awx.main.tests.models import * # noqa +from awx.main.tests.commands import * # noqa +from awx.main.tests.dbtransform import * # noqa diff --git a/awx/main/tests/base.py b/awx/main/tests/base.py index 467e30734e..69fd0e41e2 100644 --- a/awx/main/tests/base.py +++ b/awx/main/tests/base.py @@ -25,6 +25,9 @@ from django.contrib.auth.models import User from django.test.client import Client from django.test.utils import override_settings +# MongoEngine +from mongoengine.connection import get_db + # AWX from awx.main.models import * # noqa from awx.main.backend import LDAPSettings @@ -84,6 +87,11 @@ class BaseTestMixin(QueueTestMixin): def setUp(self): super(BaseTestMixin, self).setUp() + + # Drop mongo database + self.db = get_db() + self.db.connection.drop_database(settings.MONGO_DB) + self.object_ctr = 0 # Save sys.path before tests. self._sys_path = [x for x in sys.path] diff --git a/awx/main/tests/commands/__init__.py b/awx/main/tests/commands/__init__.py new file mode 100644 index 0000000000..0be275c8b8 --- /dev/null +++ b/awx/main/tests/commands/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +from awx.main.tests.commands.run_fact_cache_receiver import * # noqa +from awx.main.tests.commands.commands_monolithic import * # noqa \ No newline at end of file diff --git a/awx/main/tests/commands/base.py b/awx/main/tests/commands/base.py new file mode 100644 index 0000000000..575eb08cf4 --- /dev/null +++ b/awx/main/tests/commands/base.py @@ -0,0 +1,89 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +import StringIO +import sys +import json + +# Django +from django.core.management import call_command + +# AWX +from awx.main.models import * # noqa +from awx.main.tests.base import BaseTestMixin + +class BaseCommandMixin(BaseTestMixin): + ''' + Base class for tests that run management commands. + ''' + + def create_test_inventories(self): + self.setup_users() + self.organizations = self.make_organizations(self.super_django_user, 2) + self.projects = self.make_projects(self.normal_django_user, 2) + self.organizations[0].projects.add(self.projects[1]) + self.organizations[1].projects.add(self.projects[0]) + self.inventories = [] + self.hosts = [] + self.groups = [] + for n, organization in enumerate(self.organizations): + inventory = Inventory.objects.create(name='inventory-%d' % n, + description='description for inventory %d' % n, + organization=organization, + variables=json.dumps({'n': n}) if n else '') + self.inventories.append(inventory) + hosts = [] + for x in xrange(10): + if n > 0: + variables = json.dumps({'ho': 'hum-%d' % x}) + else: + variables = '' + host = inventory.hosts.create(name='host-%02d-%02d.example.com' % (n, x), + inventory=inventory, + variables=variables) + hosts.append(host) + self.hosts.extend(hosts) + groups = [] + for x in xrange(5): + if n > 0: + variables = json.dumps({'gee': 'whiz-%d' % x}) + else: + variables = '' + group = inventory.groups.create(name='group-%d' % x, + inventory=inventory, + variables=variables) + groups.append(group) + group.hosts.add(hosts[x]) + group.hosts.add(hosts[x + 5]) + if n > 0 and x == 4: + group.parents.add(groups[3]) + self.groups.extend(groups) + + def run_command(self, name, *args, **options): + ''' + Run a management command and capture its stdout/stderr along with any + exceptions. + ''' + command_runner = options.pop('command_runner', call_command) + stdin_fileobj = options.pop('stdin_fileobj', None) + options.setdefault('verbosity', 1) + options.setdefault('interactive', False) + original_stdin = sys.stdin + original_stdout = sys.stdout + original_stderr = sys.stderr + if stdin_fileobj: + sys.stdin = stdin_fileobj + sys.stdout = StringIO.StringIO() + sys.stderr = StringIO.StringIO() + result = None + try: + result = command_runner(name, *args, **options) + except Exception as e: + result = e + finally: + captured_stdout = sys.stdout.getvalue() + captured_stderr = sys.stderr.getvalue() + sys.stdin = original_stdin + sys.stdout = original_stdout + sys.stderr = original_stderr + return result, captured_stdout, captured_stderr diff --git a/awx/main/tests/commands.py b/awx/main/tests/commands/commands_monolithic.py similarity index 100% rename from awx/main/tests/commands.py rename to awx/main/tests/commands/commands_monolithic.py diff --git a/awx/main/tests/commands/run_fact_cache_receiver.py b/awx/main/tests/commands/run_fact_cache_receiver.py new file mode 100644 index 0000000000..c55782d015 --- /dev/null +++ b/awx/main/tests/commands/run_fact_cache_receiver.py @@ -0,0 +1,199 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +import time +from datetime import datetime +import mock +import json +import unittest +from copy import deepcopy +from mock import Mock, MagicMock + +# AWX +from awx.main.tests.base import BaseTest +from awx.main.tests.commands.base import BaseCommandMixin +from awx.main.management.commands.run_fact_cache_receiver import FactCacheReceiver, _MODULES +from awx.main.models.fact import * + +__all__ = ['RunFactCacheReceiverUnitTest', 'RunFactCacheReceiverFunctionalTest'] + +TEST_MSG_BASE = { + 'host': 'hostname1', + 'date_key': time.mktime(datetime.utcnow().timetuple()), + 'facts' : { } +} + +TEST_MSG_MODULES = { + 'packages': { + "accountsservice": [ + { + "architecture": "amd64", + "name": "accountsservice", + "source": "apt", + "version": "0.6.35-0ubuntu7.1" + } + ], + "acpid": [ + { + "architecture": "amd64", + "name": "acpid", + "source": "apt", + "version": "1:2.0.21-1ubuntu2" + } + ], + "adduser": [ + { + "architecture": "all", + "name": "adduser", + "source": "apt", + "version": "3.113+nmu3ubuntu3" + } + ], + }, + 'services': [ + { + "name": "acpid", + "source": "sysv", + "state": "running" + }, + { + "name": "apparmor", + "source": "sysv", + "state": "stopped" + }, + { + "name": "atd", + "source": "sysv", + "state": "running" + }, + { + "name": "cron", + "source": "sysv", + "state": "running" + } + ], + 'ansible': { + 'ansible_fact_simple': 'hello world', + 'ansible_fact_complex': { + 'foo': 'bar', + 'hello': [ + 'scooby', + 'dooby', + 'doo' + ] + }, + } +} +# Derived from TEST_MSG_BASE +TEST_MSG = dict(TEST_MSG_BASE) + +def copy_only_module(data, module): + data = deepcopy(data) + data['facts'] = {} + if module == 'ansible': + data['facts'] = deepcopy(TEST_MSG_MODULES[module]) + else: + data['facts'][module] = deepcopy(TEST_MSG_MODULES[module]) + return data + + +class RunFactCacheReceiverFunctionalTest(BaseCommandMixin, BaseTest): + @unittest.skip('''\ +TODO: run_fact_cache_receiver enters a while True loop that never exists. \ +This differs from most other commands that we test for. More logic and work \ +would be required to invoke this case from the command line with little return \ +in terms of increase coverage and confidence.''') + def test_invoke(self): + result, stdout, stderr = self.run_command('run_fact_cache_receiver') + self.assertEqual(result, None) + +class RunFactCacheReceiverUnitTest(BaseTest): + # TODO: Check that timestamp and other attributes are as expected + def check_process_fact_message_module(self, data, module): + fact_found = None + facts = Fact.objects.all() + self.assertEqual(len(facts), 1) + for fact in facts: + if fact.module == module: + fact_found = fact + break + self.assertIsNotNone(fact_found) + #self.assertEqual(data['facts'][module], fact_found[module]) + + fact_found = None + fact_versions = FactVersion.objects.all() + self.assertEqual(len(fact_versions), 1) + for fact in fact_versions: + if fact.module == module: + fact_found = fact + break + self.assertIsNotNone(fact_found) + + + # Ensure that the message flows from the socket through to process_fact_message() + @mock.patch('awx.main.socket.Socket.listen') + def test_run_receiver(self, listen_mock): + listen_mock.return_value = [ TEST_MSG ] + + receiver = FactCacheReceiver() + receiver.process_fact_message = MagicMock(name='process_fact_message') + receiver.run_receiver() + + receiver.process_fact_message.assert_called_once_with(TEST_MSG) + + def test_process_fact_message_ansible(self): + data = copy_only_module(TEST_MSG, 'ansible') + + receiver = FactCacheReceiver() + receiver.process_fact_message(data) + + self.check_process_fact_message_module(data, 'ansible') + + def test_process_fact_message_packages(self): + data = copy_only_module(TEST_MSG, 'packages') + + receiver = FactCacheReceiver() + receiver.process_fact_message(data) + + self.check_process_fact_message_module(data, 'packages') + + def test_process_fact_message_services(self): + data = copy_only_module(TEST_MSG, 'services') + + receiver = FactCacheReceiver() + receiver.process_fact_message(data) + + self.check_process_fact_message_module(data, 'services') + + + # Ensure that only a single host gets created for multiple invocations with the same hostname + def test_process_fact_message_single_host_created(self): + receiver = FactCacheReceiver() + + data = deepcopy(TEST_MSG) + receiver.process_fact_message(data) + data = deepcopy(TEST_MSG) + data['date_key'] = time.mktime(datetime.utcnow().timetuple()) + receiver.process_fact_message(data) + + fact_hosts = FactHost.objects.all() + self.assertEqual(len(fact_hosts), 1) + + def test_process_facts_message_ansible_overwrite(self): + data = copy_only_module(TEST_MSG, 'ansible') + key = 'ansible_overwrite' + value = 'hello world' + + receiver = FactCacheReceiver() + receiver.process_fact_message(data) + + fact = Fact.objects.all()[0] + + data = copy_only_module(TEST_MSG, 'ansible') + data['facts'][key] = value + receiver.process_fact_message(data) + + fact = Fact.objects.get(id=fact.id) + self.assertIn(key, fact.fact) + self.assertEqual(fact.fact[key], value) diff --git a/awx/main/tests/dbtransform.py b/awx/main/tests/dbtransform.py new file mode 100644 index 0000000000..47eb7d75dd --- /dev/null +++ b/awx/main/tests/dbtransform.py @@ -0,0 +1,78 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +from datetime import datetime +from mongoengine.connection import get_db +from mongoengine import connect + +# Django +from django.conf import settings + +# AWX +from awx.main.tests.base import BaseTest +from awx.main.models.fact import * + +__all__ = ['DBTransformTest'] + +class DBTransformTest(BaseTest): + def setUp(self): + super(DBTransformTest, self).setUp() + + # Create a db connection that doesn't have the transformation registered + # Note: this goes through pymongo not mongoengine + self.client = connect(settings.MONGO_DB) + self.db = self.client[settings.MONGO_DB] + + def _create_fact(self): + fact = {} + fact[self.k] = self.v + h = FactHost(hostname='blah') + h.save() + f = Fact(host=h,module='blah',timestamp=datetime.now(),fact=fact) + f.save() + return f + + def create_dot_fact(self): + self.k = 'this.is.a.key' + self.v = 'this.is.a.value' + + self.k_uni = 'this\uff0Eis\uff0Ea\uff0Ekey' + + return self._create_fact() + + def create_dollar_fact(self): + self.k = 'this$is$a$key' + self.v = 'this$is$a$value' + + self.k_uni = 'this\uff04is\uff04a\uff04key' + + return self._create_fact() + + def check_unicode(self, f): + f_raw = self.db.fact.find_one(id=f.id) + self.assertIn(self.k_uni, f_raw['fact']) + self.assertEqual(f_raw['fact'][self.k_uni], self.v) + + # Ensure key . are being transformed to the equivalent unicode into the database + def test_key_transform_dot_unicode_in_storage(self): + f = self.create_dot_fact() + self.check_unicode(f) + + # Ensure key $ are being transformed to the equivalent unicode into the database + def test_key_transform_dollar_unicode_in_storage(self): + f = self.create_dollar_fact() + self.check_unicode(f) + + def check_transform(self): + f = Fact.objects.all()[0] + self.assertIn(self.k, f.fact) + self.assertEqual(f.fact[self.k], self.v) + + def test_key_transform_dot_on_retreive(self): + self.create_dot_fact() + self.check_transform() + + def test_key_transform_dollar_on_retreive(self): + self.create_dollar_fact() + self.check_transform() \ No newline at end of file diff --git a/awx/main/tests/jobs/__init__.py b/awx/main/tests/jobs/__init__.py index 7cba799343..49c40437da 100644 --- a/awx/main/tests/jobs/__init__.py +++ b/awx/main/tests/jobs/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + from awx.main.tests.jobs.jobs_monolithic import * # noqa from survey_password import * # noqa from base import * # noqa diff --git a/awx/main/tests/models/__init__.py b/awx/main/tests/models/__init__.py new file mode 100644 index 0000000000..4ca3023f38 --- /dev/null +++ b/awx/main/tests/models/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +from awx.main.tests.models.fact import * # noqa \ No newline at end of file diff --git a/awx/main/tests/models/fact.py b/awx/main/tests/models/fact.py new file mode 100644 index 0000000000..7414f7456b --- /dev/null +++ b/awx/main/tests/models/fact.py @@ -0,0 +1,87 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +from datetime import datetime + +# Django + +# AWX +from awx.main.models.fact import * +from awx.main.tests.base import BaseTest + +__all__ = ['FactHostTest', 'FactTest'] + +TEST_FACT_DATA = { + 'hostname': 'hostname1', + 'add_fact_data': { + 'timestamp': datetime.now(), + 'host': None, + 'module': 'packages', + 'fact': { + "accountsservice": [ + { + "architecture": "amd64", + "name": "accountsservice", + "source": "apt", + "version": "0.6.35-0ubuntu7.1" + } + ], + "acpid": [ + { + "architecture": "amd64", + "name": "acpid", + "source": "apt", + "version": "1:2.0.21-1ubuntu2" + } + ], + "adduser": [ + { + "architecture": "all", + "name": "adduser", + "source": "apt", + "version": "3.113+nmu3ubuntu3" + } + ], + }, + } +} +# Strip off microseconds because mongo has less precision +TEST_FACT_DATA['add_fact_data']['timestamp'] = TEST_FACT_DATA['add_fact_data']['timestamp'].replace(microsecond=0) + +class FactHostTest(BaseTest): + def test_create_host(self): + host = FactHost(hostname=TEST_FACT_DATA['hostname']) + host.save() + + host = FactHost.objects.get(hostname=TEST_FACT_DATA['hostname']) + self.assertIsNotNone(host, "Host added but not found") + self.assertEqual(TEST_FACT_DATA['hostname'], host.hostname, "Gotten record hostname does not match expected hostname") + + +class FactTest(BaseTest): + def setUp(self): + super(FactTest, self).setUp() + TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save() + + def test_add_fact(self): + (f_obj, v_obj) = Fact.add_fact(**TEST_FACT_DATA['add_fact_data']) + f = Fact.objects.get(id=f_obj.id) + v = FactVersion.objects.get(id=v_obj.id) + + self.assertEqual(f.id, f_obj.id) + self.assertEqual(f.module, TEST_FACT_DATA['add_fact_data']['module']) + self.assertEqual(f.fact, TEST_FACT_DATA['add_fact_data']['fact']) + self.assertEqual(f.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp']) + + # host relationship created + self.assertEqual(f.host.id, TEST_FACT_DATA['add_fact_data']['host'].id) + + # version created and related + self.assertEqual(v.id, v_obj.id) + self.assertEqual(v.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp']) + self.assertEqual(v.host.id, TEST_FACT_DATA['add_fact_data']['host'].id) + self.assertEqual(v.fact.id, f_obj.id) + self.assertEqual(v.fact.module, TEST_FACT_DATA['add_fact_data']['module']) + + diff --git a/awx/plugins/fact_caching/tower.py b/awx/plugins/fact_caching/tower.py index bf6c9211da..f369d9fa79 100755 --- a/awx/plugins/fact_caching/tower.py +++ b/awx/plugins/fact_caching/tower.py @@ -32,6 +32,8 @@ import sys import time import datetime +import json +from copy import deepcopy from ansible import constants as C from ansible.cache.base import BaseCacheModule @@ -47,6 +49,7 @@ class CacheModule(BaseCacheModule): # Basic in-memory caching for typical runs self._cache = {} + self._cache_prev = {} # This is the local tower zmq connection self._tower_connection = C.CACHE_PLUGIN_CONNECTION @@ -54,20 +57,67 @@ class CacheModule(BaseCacheModule): try: self.context = zmq.Context() self.socket = self.context.socket(zmq.REQ) + self.socket.setsockopt(zmq.RCVTIMEO, 4000) + self.socket.setsockopt(zmq.LINGER, 2000) self.socket.connect(self._tower_connection) except Exception, e: print("Connection to zeromq failed at %s with error: %s" % (str(self._tower_connection), str(e))) sys.exit(1) + def identify_ansible_facts(self, facts): + ansible_keys = {} + for k in facts.keys(): + if k.startswith('ansible_'): + ansible_keys[k] = 1 + return ansible_keys + + def identify_new_module(self, key, value): + if key in self._cache_prev: + value_old = self._cache_prev[key] + for k,v in value.iteritems(): + if k not in value_old: + if not k.startswith('ansible_'): + return k + return None + def get(self, key): return self._cache.get(key) + ''' + get() returns a reference to the fact object (usually a dict). The object is modified directly, + then set is called. Effectively, pre-determining the set logic. + + The below logic creates a backup of the cache each set. The values are now preserved across set() calls. + + For a given key. The previous value is looked at for new keys that aren't of the form 'ansible_'. + If found, send the value of the found key. + If not found, send all the key value pairs of the form 'ansible_' (we presume set() is called because + of an ansible fact module invocation) + + More simply stated... + In value, if a new key is found at the top most dict then consider this a module request and only + emit the facts for the found top-level key. + + If a new key is not found, assume set() was called as a result of ansible facts scan. Thus, emit + all facts of the form 'ansible_'. + ''' def set(self, key, value): + module = self.identify_new_module(key, value) + # Assume ansible fact triggered the set if no new module found + facts = {} + if not module: + keys = self.identify_ansible_facts(value) + for k in keys: + facts[k] = value[k] + else: + facts[module] = value[module] + + self._cache_prev = deepcopy(self._cache) self._cache[key] = value # Emit fact data to tower for processing - self.socket.send_json(dict(host=key, facts=value, date_key=self.date_key)) + self.socket.send_json(dict(host=key, facts=facts, date_key=self.date_key)) self.socket.recv() def keys(self): diff --git a/awx/settings/__init__.py b/awx/settings/__init__.py index 893555cc13..73f421a6e1 100644 --- a/awx/settings/__init__.py +++ b/awx/settings/__init__.py @@ -1,2 +1,10 @@ # Copyright (c) 2014 AnsibleWorks, Inc. # All Rights Reserved. + +from django.conf import settings +from mongoengine import connect +from mongoengine.connection import get_db +from awx.main.dbtransform import register_key_transform + +connect(settings.MONGO_DB) +register_key_transform(get_db()) \ No newline at end of file diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index adf6cfd7a7..d37b047ff2 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -7,6 +7,8 @@ import glob from datetime import timedelta import tempfile +MONGO_DB = 'system_tracking' + # Update this module's local settings from the global settings module. from django.conf import global_settings this_module = sys.modules[__name__] diff --git a/awx/settings/development.py b/awx/settings/development.py index fa9e067744..d365b25f57 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -14,6 +14,8 @@ from split_settings.tools import optional, include # Load default settings. from defaults import * +MONGO_DB = 'system_tracking_dev' + # Disable capturing all SQL queries when running celeryd in development. if 'celeryd' in sys.argv: SQL_DEBUG = False diff --git a/requirements/dev_local.txt b/requirements/dev_local.txt index 7579a37993..c1c6c8420d 100644 --- a/requirements/dev_local.txt +++ b/requirements/dev_local.txt @@ -55,7 +55,6 @@ mongo-python-driver-2.8.tar.gz # Needed by pyrax: #httplib2-0.8.tar.gz #keyring-3.7.zip - #mock-1.0.1.tar.gz #python-swiftclient-2.0.3.tar.gz #rackspace-novaclient-1.4.tar.gz # Remaining dev/prod packages: @@ -78,6 +77,8 @@ mongo-python-driver-2.8.tar.gz #mongoengine-0.9.0.tar.gz # Dev-only packages: +# Needed for tests +mock-1.0.1.tar.gz # Needed by django-debug-toolbar: sqlparse-0.1.11.tar.gz # Needed for Python2.6 support: From 35e1c19fc26d50b16cc5d5115678383d94f7cb69 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 3 Apr 2015 12:00:56 -0400 Subject: [PATCH 2/7] do not run tests if mongodb connect fails --- awx/main/tests/base.py | 15 ++++++++++----- .../tests/commands/run_fact_cache_receiver.py | 7 ++++--- awx/main/tests/dbtransform.py | 4 ++-- awx/main/tests/models/fact.py | 6 +++--- awx/settings/__init__.py | 12 +++++++++--- awx/settings/local_settings.py.example | 2 ++ 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/awx/main/tests/base.py b/awx/main/tests/base.py index 69fd0e41e2..6cbbb068da 100644 --- a/awx/main/tests/base.py +++ b/awx/main/tests/base.py @@ -26,7 +26,7 @@ from django.test.client import Client from django.test.utils import override_settings # MongoEngine -from mongoengine.connection import get_db +from mongoengine.connection import get_db, ConnectionError # AWX from awx.main.models import * # noqa @@ -43,6 +43,15 @@ TEST_PLAYBOOK = '''- hosts: mygroup command: test 1 = 1 ''' +class MongoDBRequired(django.test.TestCase): + def setUp(self): + # Drop mongo database + try: + self.db = get_db() + self.db.connection.drop_database(settings.MONGO_DB) + except ConnectionError as e: + self.skipTest('MongoDB connection failed') + class QueueTestMixin(object): def start_queue(self): self.start_redis() @@ -88,10 +97,6 @@ class BaseTestMixin(QueueTestMixin): def setUp(self): super(BaseTestMixin, self).setUp() - # Drop mongo database - self.db = get_db() - self.db.connection.drop_database(settings.MONGO_DB) - self.object_ctr = 0 # Save sys.path before tests. self._sys_path = [x for x in sys.path] diff --git a/awx/main/tests/commands/run_fact_cache_receiver.py b/awx/main/tests/commands/run_fact_cache_receiver.py index c55782d015..99f0e33a8a 100644 --- a/awx/main/tests/commands/run_fact_cache_receiver.py +++ b/awx/main/tests/commands/run_fact_cache_receiver.py @@ -11,7 +11,7 @@ from copy import deepcopy from mock import Mock, MagicMock # AWX -from awx.main.tests.base import BaseTest +from awx.main.tests.base import BaseTest, MongoDBRequired from awx.main.tests.commands.base import BaseCommandMixin from awx.main.management.commands.run_fact_cache_receiver import FactCacheReceiver, _MODULES from awx.main.models.fact import * @@ -98,7 +98,7 @@ def copy_only_module(data, module): return data -class RunFactCacheReceiverFunctionalTest(BaseCommandMixin, BaseTest): +class RunFactCacheReceiverFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequired): @unittest.skip('''\ TODO: run_fact_cache_receiver enters a while True loop that never exists. \ This differs from most other commands that we test for. More logic and work \ @@ -108,7 +108,8 @@ in terms of increase coverage and confidence.''') result, stdout, stderr = self.run_command('run_fact_cache_receiver') self.assertEqual(result, None) -class RunFactCacheReceiverUnitTest(BaseTest): +class RunFactCacheReceiverUnitTest(BaseTest, MongoDBRequired): + # TODO: Check that timestamp and other attributes are as expected def check_process_fact_message_module(self, data, module): fact_found = None diff --git a/awx/main/tests/dbtransform.py b/awx/main/tests/dbtransform.py index 47eb7d75dd..164e177e13 100644 --- a/awx/main/tests/dbtransform.py +++ b/awx/main/tests/dbtransform.py @@ -10,12 +10,12 @@ from mongoengine import connect from django.conf import settings # AWX -from awx.main.tests.base import BaseTest +from awx.main.tests.base import BaseTest, MongoDBRequired from awx.main.models.fact import * __all__ = ['DBTransformTest'] -class DBTransformTest(BaseTest): +class DBTransformTest(BaseTest, MongoDBRequired): def setUp(self): super(DBTransformTest, self).setUp() diff --git a/awx/main/tests/models/fact.py b/awx/main/tests/models/fact.py index 7414f7456b..1f400080cb 100644 --- a/awx/main/tests/models/fact.py +++ b/awx/main/tests/models/fact.py @@ -8,7 +8,7 @@ from datetime import datetime # AWX from awx.main.models.fact import * -from awx.main.tests.base import BaseTest +from awx.main.tests.base import BaseTest, MongoDBRequired __all__ = ['FactHostTest', 'FactTest'] @@ -49,7 +49,7 @@ TEST_FACT_DATA = { # Strip off microseconds because mongo has less precision TEST_FACT_DATA['add_fact_data']['timestamp'] = TEST_FACT_DATA['add_fact_data']['timestamp'].replace(microsecond=0) -class FactHostTest(BaseTest): +class FactHostTest(BaseTest, MongoDBRequired): def test_create_host(self): host = FactHost(hostname=TEST_FACT_DATA['hostname']) host.save() @@ -59,7 +59,7 @@ class FactHostTest(BaseTest): self.assertEqual(TEST_FACT_DATA['hostname'], host.hostname, "Gotten record hostname does not match expected hostname") -class FactTest(BaseTest): +class FactTest(BaseTest, MongoDBRequired): def setUp(self): super(FactTest, self).setUp() TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save() diff --git a/awx/settings/__init__.py b/awx/settings/__init__.py index 73f421a6e1..988ff1d52e 100644 --- a/awx/settings/__init__.py +++ b/awx/settings/__init__.py @@ -3,8 +3,14 @@ from django.conf import settings from mongoengine import connect -from mongoengine.connection import get_db +from mongoengine.connection import get_db, ConnectionError from awx.main.dbtransform import register_key_transform +import logging -connect(settings.MONGO_DB) -register_key_transform(get_db()) \ No newline at end of file +logger = logging.getLogger('awx.settings.__init__') + +try: + connect(settings.MONGO_DB) + register_key_transform(get_db()) +except ConnectionError: + logger.warn('Failed to establish connect to MongDB "%s"' % (settings.MONGO_DB)) diff --git a/awx/settings/local_settings.py.example b/awx/settings/local_settings.py.example index db4bd8eba9..50a95e1728 100644 --- a/awx/settings/local_settings.py.example +++ b/awx/settings/local_settings.py.example @@ -42,6 +42,8 @@ if len(sys.argv) >= 2 and sys.argv[1] == 'test': 'TEST_NAME': os.path.join(BASE_DIR, 'awx_test.sqlite3'), } } + + MONGO_DB = 'system_tracking_test' # Celery AMQP configuration. BROKER_URL = 'redis://localhost/' From f5600be39e25ff42dba3768f8022514bf5c0ba4d Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 3 Apr 2015 12:54:56 -0400 Subject: [PATCH 3/7] missed changing path to external resources --- awx/main/tests/commands/commands_monolithic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/awx/main/tests/commands/commands_monolithic.py b/awx/main/tests/commands/commands_monolithic.py index 7f78af5da3..b26ee67d0e 100644 --- a/awx/main/tests/commands/commands_monolithic.py +++ b/awx/main/tests/commands/commands_monolithic.py @@ -870,7 +870,7 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest): parts.query, parts.fragment]) os.environ.setdefault('REST_API_URL', rest_api_url) os.environ['INVENTORY_ID'] = str(old_inv.pk) - source = os.path.join(os.path.dirname(__file__), '..', '..', 'plugins', + source = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'plugins', 'inventory', 'awxrest.py') result, stdout, stderr = self.run_command('inventory_import', inventory_id=new_inv.pk, @@ -907,7 +907,7 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest): new_inv = self.organizations[0].inventories.create(name='newec2') self.assertEqual(new_inv.hosts.count(), 0) self.assertEqual(new_inv.groups.count(), 0) - os.chdir(os.path.join(os.path.dirname(__file__), 'data')) + os.chdir(os.path.join(os.path.dirname(__file__), '..', 'data')) inv_file = 'large_ec2_inventory.py' result, stdout, stderr = self.run_command('inventory_import', inventory_id=new_inv.pk, @@ -928,7 +928,7 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest): new_inv = self.organizations[0].inventories.create(name='splunk') self.assertEqual(new_inv.hosts.count(), 0) self.assertEqual(new_inv.groups.count(), 0) - inv_file = os.path.join(os.path.dirname(__file__), 'data', + inv_file = os.path.join(os.path.dirname(__file__), '..', 'data', 'splunk_inventory.py') result, stdout, stderr = self.run_command('inventory_import', inventory_id=new_inv.pk, @@ -951,7 +951,7 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest): def _check_largeinv_import(self, new_inv, nhosts, nhosts_inactive=0): self._start_time = time.time() - inv_file = os.path.join(os.path.dirname(__file__), 'data', 'largeinv.py') + inv_file = os.path.join(os.path.dirname(__file__), '..', 'data', 'largeinv.py') ngroups = self._get_ngroups_for_nhosts(nhosts) os.environ['NHOSTS'] = str(nhosts) result, stdout, stderr = self.run_command('inventory_import', From a769af0e830a8049772ddf321d769f58f7327f8d Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 3 Apr 2015 14:55:37 -0400 Subject: [PATCH 4/7] flake8 fixes --- .../commands/run_fact_cache_receiver.py | 10 ++--- awx/main/models/fact.py | 8 ++-- awx/main/tests/base.py | 2 +- awx/main/tests/commands/__init__.py | 2 +- .../tests/commands/run_fact_cache_receiver.py | 45 +++++++++---------- awx/main/tests/dbtransform.py | 5 +-- awx/main/tests/models/__init__.py | 2 +- awx/main/tests/models/fact.py | 38 ++++++++-------- 8 files changed, 53 insertions(+), 59 deletions(-) diff --git a/awx/main/management/commands/run_fact_cache_receiver.py b/awx/main/management/commands/run_fact_cache_receiver.py index f31a8495a1..9ab51a0c93 100644 --- a/awx/main/management/commands/run_fact_cache_receiver.py +++ b/awx/main/management/commands/run_fact_cache_receiver.py @@ -3,17 +3,13 @@ import logging from datetime import datetime -import json from django.core.management.base import NoArgsCommand from awx.main.models import * # noqa from awx.main.socket import Socket -import pymongo -from pymongo import MongoClient - -_MODULES = [ 'packages', 'services', 'files' ] +_MODULES = ['packages', 'services', 'files'] logger = logging.getLogger('awx.main.commands.run_fact_cache_receiver') class FactCacheReceiver(object): @@ -49,10 +45,10 @@ class FactCacheReceiver(object): try: host = FactHost.objects.get(hostname=hostname) - except FactHost.DoesNotExist as e: + except FactHost.DoesNotExist: host = FactHost(hostname=hostname) host.save() - except FactHost.MultipleObjectsReturned as e: + except FactHost.MultipleObjectsReturned: query = "db['fact_host'].find(hostname=%s)" % hostname print('Database inconsistent. Multiple FactHost "%s" exist. Try the query %s to find the records.' % (hostname, query)) return diff --git a/awx/main/models/fact.py b/awx/main/models/fact.py index b97b11d7e3..6e482a0b80 100644 --- a/awx/main/models/fact.py +++ b/awx/main/models/fact.py @@ -1,15 +1,15 @@ from mongoengine import Document, DynamicDocument, DateTimeField, ReferenceField, StringField class FactHost(Document): - hostname = StringField(max_length=100, required=True, unique=True) + hostname = StringField(max_length=100, required=True, unique=True) # TODO: Consider using hashed index on hostname. django-mongo may not support this but # executing raw js will - meta = { + meta = { 'indexes': [ - 'hostname' + 'hostname' ] - } + } class Fact(DynamicDocument): timestamp = DateTimeField(required=True) diff --git a/awx/main/tests/base.py b/awx/main/tests/base.py index 6cbbb068da..7f96c818cc 100644 --- a/awx/main/tests/base.py +++ b/awx/main/tests/base.py @@ -49,7 +49,7 @@ class MongoDBRequired(django.test.TestCase): try: self.db = get_db() self.db.connection.drop_database(settings.MONGO_DB) - except ConnectionError as e: + except ConnectionError: self.skipTest('MongoDB connection failed') class QueueTestMixin(object): diff --git a/awx/main/tests/commands/__init__.py b/awx/main/tests/commands/__init__.py index 0be275c8b8..7a1446f52a 100644 --- a/awx/main/tests/commands/__init__.py +++ b/awx/main/tests/commands/__init__.py @@ -2,4 +2,4 @@ # All Rights Reserved from awx.main.tests.commands.run_fact_cache_receiver import * # noqa -from awx.main.tests.commands.commands_monolithic import * # noqa \ No newline at end of file +from awx.main.tests.commands.commands_monolithic import * # noqa diff --git a/awx/main/tests/commands/run_fact_cache_receiver.py b/awx/main/tests/commands/run_fact_cache_receiver.py index 99f0e33a8a..d57dde93c7 100644 --- a/awx/main/tests/commands/run_fact_cache_receiver.py +++ b/awx/main/tests/commands/run_fact_cache_receiver.py @@ -5,16 +5,15 @@ import time from datetime import datetime import mock -import json import unittest from copy import deepcopy -from mock import Mock, MagicMock +from mock import MagicMock # AWX from awx.main.tests.base import BaseTest, MongoDBRequired from awx.main.tests.commands.base import BaseCommandMixin -from awx.main.management.commands.run_fact_cache_receiver import FactCacheReceiver, _MODULES -from awx.main.models.fact import * +from awx.main.management.commands.run_fact_cache_receiver import FactCacheReceiver +from awx.main.models.fact import * # noqa __all__ = ['RunFactCacheReceiverUnitTest', 'RunFactCacheReceiverFunctionalTest'] @@ -27,28 +26,28 @@ TEST_MSG_BASE = { TEST_MSG_MODULES = { 'packages': { "accountsservice": [ - { - "architecture": "amd64", - "name": "accountsservice", - "source": "apt", - "version": "0.6.35-0ubuntu7.1" - } + { + "architecture": "amd64", + "name": "accountsservice", + "source": "apt", + "version": "0.6.35-0ubuntu7.1" + } ], "acpid": [ - { - "architecture": "amd64", - "name": "acpid", - "source": "apt", - "version": "1:2.0.21-1ubuntu2" - } + { + "architecture": "amd64", + "name": "acpid", + "source": "apt", + "version": "1:2.0.21-1ubuntu2" + } ], "adduser": [ - { - "architecture": "all", - "name": "adduser", - "source": "apt", - "version": "3.113+nmu3ubuntu3" - } + { + "architecture": "all", + "name": "adduser", + "source": "apt", + "version": "3.113+nmu3ubuntu3" + } ], }, 'services': [ @@ -135,7 +134,7 @@ class RunFactCacheReceiverUnitTest(BaseTest, MongoDBRequired): # Ensure that the message flows from the socket through to process_fact_message() @mock.patch('awx.main.socket.Socket.listen') def test_run_receiver(self, listen_mock): - listen_mock.return_value = [ TEST_MSG ] + listen_mock.return_value = [TEST_MSG] receiver = FactCacheReceiver() receiver.process_fact_message = MagicMock(name='process_fact_message') diff --git a/awx/main/tests/dbtransform.py b/awx/main/tests/dbtransform.py index 164e177e13..1c18098118 100644 --- a/awx/main/tests/dbtransform.py +++ b/awx/main/tests/dbtransform.py @@ -3,7 +3,6 @@ # Python from datetime import datetime -from mongoengine.connection import get_db from mongoengine import connect # Django @@ -11,7 +10,7 @@ from django.conf import settings # AWX from awx.main.tests.base import BaseTest, MongoDBRequired -from awx.main.models.fact import * +from awx.main.models.fact import * # noqa __all__ = ['DBTransformTest'] @@ -75,4 +74,4 @@ class DBTransformTest(BaseTest, MongoDBRequired): def test_key_transform_dollar_on_retreive(self): self.create_dollar_fact() - self.check_transform() \ No newline at end of file + self.check_transform() diff --git a/awx/main/tests/models/__init__.py b/awx/main/tests/models/__init__.py index 4ca3023f38..a4959c9e29 100644 --- a/awx/main/tests/models/__init__.py +++ b/awx/main/tests/models/__init__.py @@ -1,4 +1,4 @@ # Copyright (c) 2015 Ansible, Inc. # All Rights Reserved -from awx.main.tests.models.fact import * # noqa \ No newline at end of file +from awx.main.tests.models.fact import * # noqa diff --git a/awx/main/tests/models/fact.py b/awx/main/tests/models/fact.py index 1f400080cb..9dbbd23b54 100644 --- a/awx/main/tests/models/fact.py +++ b/awx/main/tests/models/fact.py @@ -7,7 +7,7 @@ from datetime import datetime # Django # AWX -from awx.main.models.fact import * +from awx.main.models.fact import * # noqa from awx.main.tests.base import BaseTest, MongoDBRequired __all__ = ['FactHostTest', 'FactTest'] @@ -20,28 +20,28 @@ TEST_FACT_DATA = { 'module': 'packages', 'fact': { "accountsservice": [ - { - "architecture": "amd64", - "name": "accountsservice", - "source": "apt", - "version": "0.6.35-0ubuntu7.1" - } + { + "architecture": "amd64", + "name": "accountsservice", + "source": "apt", + "version": "0.6.35-0ubuntu7.1" + } ], "acpid": [ - { - "architecture": "amd64", - "name": "acpid", - "source": "apt", - "version": "1:2.0.21-1ubuntu2" - } + { + "architecture": "amd64", + "name": "acpid", + "source": "apt", + "version": "1:2.0.21-1ubuntu2" + } ], "adduser": [ - { - "architecture": "all", - "name": "adduser", - "source": "apt", - "version": "3.113+nmu3ubuntu3" - } + { + "architecture": "all", + "name": "adduser", + "source": "apt", + "version": "3.113+nmu3ubuntu3" + } ], }, } From d93a2ec9d14bf3ddc3ae7fbd0559a76918c782dd Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 6 Apr 2015 09:19:23 -0400 Subject: [PATCH 5/7] added host fact query in time --- awx/main/models/fact.py | 31 +++++++++++++++++++++++++++- awx/main/tests/models/fact.py | 39 ++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/awx/main/models/fact.py b/awx/main/models/fact.py index 6e482a0b80..e8f57b0463 100644 --- a/awx/main/models/fact.py +++ b/awx/main/models/fact.py @@ -11,6 +11,13 @@ class FactHost(Document): ] } + @staticmethod + def get_host_id(hostname): + host = FactHost.objects.get(hostname=hostname) + if host: + return host.id + return None + class Fact(DynamicDocument): timestamp = DateTimeField(required=True) host = ReferenceField(FactHost, required=True) @@ -34,6 +41,28 @@ class Fact(DynamicDocument): version_obj.save() return (fact_obj, version_obj) + @staticmethod + def get_version(hostname, timestamp, module=None): + try: + host = FactHost.objects.get(hostname=hostname) + except FactHost.DoesNotExist: + return None + + kv = { + 'host' : host.id, + 'timestamp__lte': timestamp + } + if module: + kv['module'] = module + + try: + facts = Fact.objects.filter(**kv) + if not facts: + return None + return facts[0] + except Fact.DoesNotExist: + return None + class FactVersion(Document): timestamp = DateTimeField(required=True) host = ReferenceField(FactHost, required=True) @@ -46,4 +75,4 @@ class FactVersion(Document): '-timestamp', 'module' ] - } + } \ No newline at end of file diff --git a/awx/main/tests/models/fact.py b/awx/main/tests/models/fact.py index 9dbbd23b54..8a69262952 100644 --- a/awx/main/tests/models/fact.py +++ b/awx/main/tests/models/fact.py @@ -3,6 +3,7 @@ # Python from datetime import datetime +from copy import deepcopy # Django @@ -10,7 +11,7 @@ from datetime import datetime from awx.main.models.fact import * # noqa from awx.main.tests.base import BaseTest, MongoDBRequired -__all__ = ['FactHostTest', 'FactTest'] +__all__ = ['FactHostTest', 'FactTest', 'FactGetVersionTest'] TEST_FACT_DATA = { 'hostname': 'hostname1', @@ -58,6 +59,12 @@ class FactHostTest(BaseTest, MongoDBRequired): self.assertIsNotNone(host, "Host added but not found") self.assertEqual(TEST_FACT_DATA['hostname'], host.hostname, "Gotten record hostname does not match expected hostname") + # Ensure an error is raised for .get() that doesn't match a record. + def test_get_host_id_no_result(self): + host = FactHost(hostname=TEST_FACT_DATA['hostname']) + host.save() + + self.assertRaises(FactHost.DoesNotExist, FactHost.objects.get, hostname='doesnotexist') class FactTest(BaseTest, MongoDBRequired): def setUp(self): @@ -84,4 +91,34 @@ class FactTest(BaseTest, MongoDBRequired): self.assertEqual(v.fact.id, f_obj.id) self.assertEqual(v.fact.module, TEST_FACT_DATA['add_fact_data']['module']) +class FactGetVersionTest(BaseTest, MongoDBRequired): + def setUp(self): + super(FactGetVersionTest, self).setUp() + TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save() + + self.t1 = datetime.now().replace(second=1, microsecond=0) + self.t2 = datetime.now().replace(second=2, microsecond=0) + data = deepcopy(TEST_FACT_DATA) + data['add_fact_data']['timestamp'] = self.t1 + (self.f1, self.v1) = Fact.add_fact(**data['add_fact_data']) + data = deepcopy(TEST_FACT_DATA) + data['add_fact_data']['timestamp'] = self.t2 + (self.f2, self.v2) = Fact.add_fact(**data['add_fact_data']) + + def test_get_version_exact_timestamp(self): + fact = Fact.get_version(hostname=TEST_FACT_DATA['hostname'], timestamp=self.t1, module=TEST_FACT_DATA['add_fact_data']['module']) + self.assertIsNotNone(fact, "Set of Facts not found") + self.assertEqual(self.f1.id, fact.id) + self.assertEqual(self.f1.fact, fact.fact) + + def test_get_version_lte_timestamp(self): + t3 = datetime.now().replace(second=3, microsecond=0) + fact = Fact.get_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module']) + self.assertEqual(self.f1.id, fact.id) + self.assertEqual(self.f1.fact, fact.fact) + + def test_get_version_none(self): + t3 = deepcopy(self.t1).replace(second=0) + fact = Fact.get_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module']) + self.assertIsNone(fact) From 85c753afea77b544b59d4551d18822d14e18b5d3 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 6 Apr 2015 10:32:23 -0400 Subject: [PATCH 6/7] added host timeline query --- awx/main/models/fact.py | 27 ++++++++++++++---- awx/main/tests/models/fact.py | 52 +++++++++++++++++++++++++++-------- 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/awx/main/models/fact.py b/awx/main/models/fact.py index e8f57b0463..0887fca171 100644 --- a/awx/main/models/fact.py +++ b/awx/main/models/fact.py @@ -41,8 +41,11 @@ class Fact(DynamicDocument): version_obj.save() return (fact_obj, version_obj) + # TODO: if we want to relax the need to include module... + # If module not specified then filter query may return more than 1 result. + # Thus, the resulting facts must somehow be unioned/concated/ or kept as an array. @staticmethod - def get_version(hostname, timestamp, module=None): + def get_host_version(hostname, timestamp, module): try: host = FactHost.objects.get(hostname=hostname) except FactHost.DoesNotExist: @@ -50,10 +53,9 @@ class Fact(DynamicDocument): kv = { 'host' : host.id, - 'timestamp__lte': timestamp + 'timestamp__lte': timestamp, + 'module': module, } - if module: - kv['module'] = module try: facts = Fact.objects.filter(**kv) @@ -63,6 +65,20 @@ class Fact(DynamicDocument): except Fact.DoesNotExist: return None + @staticmethod + def get_host_timeline(hostname, module): + try: + host = FactHost.objects.get(hostname=hostname) + except FactHost.DoesNotExist: + return None + + kv = { + 'host': host.id, + 'module': module, + } + + return FactVersion.objects.filter(**kv).values_list('timestamp') + class FactVersion(Document): timestamp = DateTimeField(required=True) host = ReferenceField(FactHost, required=True) @@ -75,4 +91,5 @@ class FactVersion(Document): '-timestamp', 'module' ] - } \ No newline at end of file + } + \ No newline at end of file diff --git a/awx/main/tests/models/fact.py b/awx/main/tests/models/fact.py index 8a69262952..63a0d98ae1 100644 --- a/awx/main/tests/models/fact.py +++ b/awx/main/tests/models/fact.py @@ -11,7 +11,7 @@ from copy import deepcopy from awx.main.models.fact import * # noqa from awx.main.tests.base import BaseTest, MongoDBRequired -__all__ = ['FactHostTest', 'FactTest', 'FactGetVersionTest'] +__all__ = ['FactHostTest', 'FactTest', 'FactGetHostVersionTest', 'FactGetHostTimeline'] TEST_FACT_DATA = { 'hostname': 'hostname1', @@ -50,6 +50,21 @@ TEST_FACT_DATA = { # Strip off microseconds because mongo has less precision TEST_FACT_DATA['add_fact_data']['timestamp'] = TEST_FACT_DATA['add_fact_data']['timestamp'].replace(microsecond=0) +def create_host_document(): + TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save() + +def create_fact_scans(count=1): + timestamps = [] + for i in range(0, count): + data = deepcopy(TEST_FACT_DATA) + t = datetime.now().replace(year=2015 - i, microsecond=0) + data['add_fact_data']['timestamp'] = t + (f, v) = Fact.add_fact(**data['add_fact_data']) + timestamps.append(t) + + return timestamps + + class FactHostTest(BaseTest, MongoDBRequired): def test_create_host(self): host = FactHost(hostname=TEST_FACT_DATA['hostname']) @@ -69,7 +84,7 @@ class FactHostTest(BaseTest, MongoDBRequired): class FactTest(BaseTest, MongoDBRequired): def setUp(self): super(FactTest, self).setUp() - TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save() + create_host_document() def test_add_fact(self): (f_obj, v_obj) = Fact.add_fact(**TEST_FACT_DATA['add_fact_data']) @@ -91,10 +106,10 @@ class FactTest(BaseTest, MongoDBRequired): self.assertEqual(v.fact.id, f_obj.id) self.assertEqual(v.fact.module, TEST_FACT_DATA['add_fact_data']['module']) -class FactGetVersionTest(BaseTest, MongoDBRequired): +class FactGetHostVersionTest(BaseTest, MongoDBRequired): def setUp(self): - super(FactGetVersionTest, self).setUp() - TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save() + super(FactGetHostVersionTest, self).setUp() + create_host_document() self.t1 = datetime.now().replace(second=1, microsecond=0) self.t2 = datetime.now().replace(second=2, microsecond=0) @@ -105,20 +120,35 @@ class FactGetVersionTest(BaseTest, MongoDBRequired): data['add_fact_data']['timestamp'] = self.t2 (self.f2, self.v2) = Fact.add_fact(**data['add_fact_data']) - def test_get_version_exact_timestamp(self): - fact = Fact.get_version(hostname=TEST_FACT_DATA['hostname'], timestamp=self.t1, module=TEST_FACT_DATA['add_fact_data']['module']) + def test_get_host_version_exact_timestamp(self): + fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=self.t1, module=TEST_FACT_DATA['add_fact_data']['module']) self.assertIsNotNone(fact, "Set of Facts not found") self.assertEqual(self.f1.id, fact.id) self.assertEqual(self.f1.fact, fact.fact) - def test_get_version_lte_timestamp(self): + def test_get_host_version_lte_timestamp(self): t3 = datetime.now().replace(second=3, microsecond=0) - fact = Fact.get_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module']) + fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module']) self.assertEqual(self.f1.id, fact.id) self.assertEqual(self.f1.fact, fact.fact) - def test_get_version_none(self): + def test_get_host_version_none(self): t3 = deepcopy(self.t1).replace(second=0) - fact = Fact.get_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module']) + fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module']) self.assertIsNone(fact) +class FactGetHostTimeline(BaseTest, MongoDBRequired): + def setUp(self): + super(FactGetHostTimeline, self).setUp() + create_host_document() + + self.scans = 20 + self.timestamps = create_fact_scans(self.scans) + + def test_get_host_timeline_ok(self): + timestamps = Fact.get_host_timeline(hostname=TEST_FACT_DATA['hostname'], module=TEST_FACT_DATA['add_fact_data']['module']) + self.assertIsNotNone(timestamps) + self.assertEqual(len(timestamps), len(self.timestamps)) + for i in range(0, self.scans): + self.assertEqual(timestamps[i], self.timestamps[i]) + From 2a039bb31f792b8acd1584a68339211f5e9bf4f5 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 6 Apr 2015 12:29:11 -0400 Subject: [PATCH 7/7] moved new fact implementation to fact app --- awx/fact/__init__.py | 20 +++++++++++++++++++ awx/fact/models/__init__.py | 6 ++++++ awx/{main => fact}/models/fact.py | 0 awx/fact/tests/__init__.py | 7 +++++++ awx/fact/tests/models/__init__.py | 6 ++++++ awx/{main => fact}/tests/models/fact.py | 2 +- awx/fact/tests/utils/__init__.py | 6 ++++++ .../tests => fact/tests/utils}/dbtransform.py | 2 +- .../tests/models => fact/utils}/__init__.py | 2 -- awx/{main => fact/utils}/dbtransform.py | 0 .../commands/run_fact_cache_receiver.py | 2 +- awx/main/models/__init__.py | 1 - awx/main/tests/__init__.py | 2 -- .../tests/commands/run_fact_cache_receiver.py | 2 +- awx/settings/__init__.py | 14 ------------- awx/settings/defaults.py | 1 + awx/settings/development.py | 2 +- 17 files changed, 51 insertions(+), 24 deletions(-) create mode 100644 awx/fact/__init__.py create mode 100644 awx/fact/models/__init__.py rename awx/{main => fact}/models/fact.py (100%) create mode 100644 awx/fact/tests/__init__.py create mode 100644 awx/fact/tests/models/__init__.py rename awx/{main => fact}/tests/models/fact.py (99%) create mode 100644 awx/fact/tests/utils/__init__.py rename awx/{main/tests => fact/tests/utils}/dbtransform.py (98%) rename awx/{main/tests/models => fact/utils}/__init__.py (53%) rename awx/{main => fact/utils}/dbtransform.py (100%) diff --git a/awx/fact/__init__.py b/awx/fact/__init__.py new file mode 100644 index 0000000000..8ce7a9e682 --- /dev/null +++ b/awx/fact/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) 2014 AnsibleWorks, Inc. +# All Rights Reserved. + +from __future__ import absolute_import + +import logging +from django.conf import settings + +from mongoengine import connect +from mongoengine.connection import get_db, ConnectionError +from .utils.dbtransform import register_key_transform + +logger = logging.getLogger('fact.__init__') + +# Connect to Mongo +try: + connect(settings.MONGO_DB) + register_key_transform(get_db()) +except ConnectionError: + logger.warn('Failed to establish connect to MongDB "%s"' % (settings.MONGO_DB)) diff --git a/awx/fact/models/__init__.py b/awx/fact/models/__init__.py new file mode 100644 index 0000000000..049720a11a --- /dev/null +++ b/awx/fact/models/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +from __future__ import absolute_import + +from .fact import * # noqa diff --git a/awx/main/models/fact.py b/awx/fact/models/fact.py similarity index 100% rename from awx/main/models/fact.py rename to awx/fact/models/fact.py diff --git a/awx/fact/tests/__init__.py b/awx/fact/tests/__init__.py new file mode 100644 index 0000000000..d7187f3928 --- /dev/null +++ b/awx/fact/tests/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +from __future__ import absolute_import + +from .models import * # noqa +from .utils import * # noqa diff --git a/awx/fact/tests/models/__init__.py b/awx/fact/tests/models/__init__.py new file mode 100644 index 0000000000..049720a11a --- /dev/null +++ b/awx/fact/tests/models/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +from __future__ import absolute_import + +from .fact import * # noqa diff --git a/awx/main/tests/models/fact.py b/awx/fact/tests/models/fact.py similarity index 99% rename from awx/main/tests/models/fact.py rename to awx/fact/tests/models/fact.py index 63a0d98ae1..ffcd38857d 100644 --- a/awx/main/tests/models/fact.py +++ b/awx/fact/tests/models/fact.py @@ -8,7 +8,7 @@ from copy import deepcopy # Django # AWX -from awx.main.models.fact import * # noqa +from awx.fact.models.fact import * # noqa from awx.main.tests.base import BaseTest, MongoDBRequired __all__ = ['FactHostTest', 'FactTest', 'FactGetHostVersionTest', 'FactGetHostTimeline'] diff --git a/awx/fact/tests/utils/__init__.py b/awx/fact/tests/utils/__init__.py new file mode 100644 index 0000000000..80e83d4661 --- /dev/null +++ b/awx/fact/tests/utils/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +from __future__ import absolute_import + +from .dbtransform import * # noqa diff --git a/awx/main/tests/dbtransform.py b/awx/fact/tests/utils/dbtransform.py similarity index 98% rename from awx/main/tests/dbtransform.py rename to awx/fact/tests/utils/dbtransform.py index 1c18098118..a11375223d 100644 --- a/awx/main/tests/dbtransform.py +++ b/awx/fact/tests/utils/dbtransform.py @@ -10,7 +10,7 @@ from django.conf import settings # AWX from awx.main.tests.base import BaseTest, MongoDBRequired -from awx.main.models.fact import * # noqa +from awx.fact.models.fact import * # noqa __all__ = ['DBTransformTest'] diff --git a/awx/main/tests/models/__init__.py b/awx/fact/utils/__init__.py similarity index 53% rename from awx/main/tests/models/__init__.py rename to awx/fact/utils/__init__.py index a4959c9e29..3a75c16036 100644 --- a/awx/main/tests/models/__init__.py +++ b/awx/fact/utils/__init__.py @@ -1,4 +1,2 @@ # Copyright (c) 2015 Ansible, Inc. # All Rights Reserved - -from awx.main.tests.models.fact import * # noqa diff --git a/awx/main/dbtransform.py b/awx/fact/utils/dbtransform.py similarity index 100% rename from awx/main/dbtransform.py rename to awx/fact/utils/dbtransform.py diff --git a/awx/main/management/commands/run_fact_cache_receiver.py b/awx/main/management/commands/run_fact_cache_receiver.py index 9ab51a0c93..2330a449e8 100644 --- a/awx/main/management/commands/run_fact_cache_receiver.py +++ b/awx/main/management/commands/run_fact_cache_receiver.py @@ -6,7 +6,7 @@ from datetime import datetime from django.core.management.base import NoArgsCommand -from awx.main.models import * # noqa +from awx.fact.models.fact import * # noqa from awx.main.socket import Socket _MODULES = ['packages', 'services', 'files'] diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 9d89df393e..b7d57212de 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -16,7 +16,6 @@ from awx.main.models.ad_hoc_commands import * # noqa from awx.main.models.schedules import * # noqa from awx.main.models.activity_stream import * # noqa from awx.main.models.ha import * # noqa -from awx.main.models.fact import * # noqa # Monkeypatch Django serializer to ignore django-taggit fields (which break # the dumpdata command; see https://github.com/alex/django-taggit/issues/155). diff --git a/awx/main/tests/__init__.py b/awx/main/tests/__init__.py index 29d772d495..dda898b544 100644 --- a/awx/main/tests/__init__.py +++ b/awx/main/tests/__init__.py @@ -15,6 +15,4 @@ from awx.main.tests.activity_stream import * # noqa from awx.main.tests.schedules import * # noqa from awx.main.tests.redact import * # noqa from awx.main.tests.views import * # noqa -from awx.main.tests.models import * # noqa from awx.main.tests.commands import * # noqa -from awx.main.tests.dbtransform import * # noqa diff --git a/awx/main/tests/commands/run_fact_cache_receiver.py b/awx/main/tests/commands/run_fact_cache_receiver.py index d57dde93c7..1a63b25180 100644 --- a/awx/main/tests/commands/run_fact_cache_receiver.py +++ b/awx/main/tests/commands/run_fact_cache_receiver.py @@ -13,7 +13,7 @@ from mock import MagicMock from awx.main.tests.base import BaseTest, MongoDBRequired from awx.main.tests.commands.base import BaseCommandMixin from awx.main.management.commands.run_fact_cache_receiver import FactCacheReceiver -from awx.main.models.fact import * # noqa +from awx.fact.models.fact import * # noqa __all__ = ['RunFactCacheReceiverUnitTest', 'RunFactCacheReceiverFunctionalTest'] diff --git a/awx/settings/__init__.py b/awx/settings/__init__.py index 988ff1d52e..893555cc13 100644 --- a/awx/settings/__init__.py +++ b/awx/settings/__init__.py @@ -1,16 +1,2 @@ # Copyright (c) 2014 AnsibleWorks, Inc. # All Rights Reserved. - -from django.conf import settings -from mongoengine import connect -from mongoengine.connection import get_db, ConnectionError -from awx.main.dbtransform import register_key_transform -import logging - -logger = logging.getLogger('awx.settings.__init__') - -try: - connect(settings.MONGO_DB) - register_key_transform(get_db()) -except ConnectionError: - logger.warn('Failed to establish connect to MongDB "%s"' % (settings.MONGO_DB)) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index d37b047ff2..3610879c02 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -155,6 +155,7 @@ INSTALLED_APPS = ( 'awx.main', 'awx.api', 'awx.ui', + 'awx.fact', ) INTERNAL_IPS = ('127.0.0.1',) diff --git a/awx/settings/development.py b/awx/settings/development.py index d365b25f57..75922c0081 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -32,7 +32,7 @@ AWX_PROOT_ENABLED = True try: import django_jenkins INSTALLED_APPS += ('django_jenkins',) - PROJECT_APPS = ('awx.main.tests', 'awx.api.tests',) + PROJECT_APPS = ('awx.main.tests', 'awx.api.tests', 'awx.fact.tests',) except ImportError: pass