diff --git a/awx/main/dbtransform.py b/awx/main/dbtransform.py new file mode 100644 index 0000000000..f88708a467 --- /dev/null +++ b/awx/main/dbtransform.py @@ -0,0 +1,55 @@ +# Copyright (c) 2014, Ansible, Inc. +# All Rights Reserved. +from pymongo.son_manipulator import SONManipulator + +''' +Inspired by: https://stackoverflow.com/questions/8429318/how-to-use-dot-in-field-name/20698802#20698802 + +Replace . and $ with unicode values +''' +class KeyTransform(SONManipulator): + def __init__(self, replace): + self.replace = replace + + def transform_key(self, key, replace, replacement): + """Transform key for saving to database.""" + return key.replace(replace, replacement) + + def revert_key(self, key, replace, replacement): + """Restore transformed key returning from database.""" + return key.replace(replacement, replace) + + def transform_incoming(self, son, collection): + """Recursively replace all keys that need transforming.""" + for (key, value) in son.items(): + for r in self.replace: + replace = r[0] + replacement = r[1] + if replace in key: + if isinstance(value, dict): + son[self.transform_key(key, replace, replacement)] = self.transform_incoming( + son.pop(key), collection) + else: + son[self.transform_key(key, replace, replacement)] = son.pop(key) + elif isinstance(value, dict): # recurse into sub-docs + son[key] = self.transform_incoming(value, collection) + return son + + def transform_outgoing(self, son, collection): + """Recursively restore all transformed keys.""" + for (key, value) in son.items(): + for r in self.replace: + replace = r[0] + replacement = r[1] + if replacement in key: + if isinstance(value, dict): + son[self.revert_key(key, replace, replacement)] = self.transform_outgoing( + son.pop(key), collection) + else: + son[self.revert_key(key, replace, replacement)] = son.pop(key) + elif isinstance(value, dict): # recurse into sub-docs + son[key] = self.transform_outgoing(value, collection) + return son + +def register_key_transform(db): + db.add_son_manipulator(KeyTransform([('.', '\uff0E'), ('$', '\uff04')])) diff --git a/awx/main/management/commands/run_fact_cache_receiver.py b/awx/main/management/commands/run_fact_cache_receiver.py index 827ce5adaa..f31a8495a1 100644 --- a/awx/main/management/commands/run_fact_cache_receiver.py +++ b/awx/main/management/commands/run_fact_cache_receiver.py @@ -2,88 +2,71 @@ # All Rights Reserved import logging +from datetime import datetime +import json from django.core.management.base import NoArgsCommand from awx.main.models import * # noqa from awx.main.socket import Socket +import pymongo from pymongo import MongoClient +_MODULES = [ 'packages', 'services', 'files' ] + logger = logging.getLogger('awx.main.commands.run_fact_cache_receiver') - -from pymongo.son_manipulator import SONManipulator - -class KeyTransform(SONManipulator): - """Transforms keys going to database and restores them coming out. - - This allows keys with dots in them to be used (but does break searching on - them unless the find command also uses the transform. - - Example & test: - # To allow `.` (dots) in keys - import pymongo - client = pymongo.MongoClient("mongodb://localhost") - db = client['delete_me'] - db.add_son_manipulator(KeyTransform(".", "_dot_")) - db['mycol'].remove() - db['mycol'].update({'_id': 1}, {'127.0.0.1': 'localhost'}, upsert=True, - manipulate=True) - print db['mycol'].find().next() - print db['mycol'].find({'127_dot_0_dot_0_dot_1': 'localhost'}).next() - - Note: transformation could be easily extended to be more complex. - """ - - def __init__(self, replace, replacement): - self.replace = replace - self.replacement = replacement - - def transform_key(self, key): - """Transform key for saving to database.""" - return key.replace(self.replace, self.replacement) - - def revert_key(self, key): - """Restore transformed key returning from database.""" - return key.replace(self.replacement, self.replace) - - def transform_incoming(self, son, collection): - """Recursively replace all keys that need transforming.""" - for (key, value) in son.items(): - if self.replace in key: - if isinstance(value, dict): - son[self.transform_key(key)] = self.transform_incoming( - son.pop(key), collection) - else: - son[self.transform_key(key)] = son.pop(key) - elif isinstance(value, dict): # recurse into sub-docs - son[key] = self.transform_incoming(value, collection) - return son - - def transform_outgoing(self, son, collection): - return son - class FactCacheReceiver(object): - def __init__(self): - self.client = MongoClient('localhost', 27017) - + self.timestamp = None + + def _determine_module(self, facts): + for x in _MODULES: + if x in facts: + return x + return 'ansible' + + def _extract_module_facts(self, module, facts): + if module in facts: + f = facts[module] + return f + return facts + + def process_facts(self, facts): + module = self._determine_module(facts) + facts = self._extract_module_facts(module, facts) + return (module, facts) + def process_fact_message(self, message): - host = message['host'].replace(".", "_") - facts = message['facts'] + hostname = message['host'] + facts_data = message['facts'] date_key = message['date_key'] - host_db = self.client.host_facts - host_db.add_son_manipulator(KeyTransform(".", "_")) - host_db.add_son_manipulator(KeyTransform("$", "_")) - host_collection = host_db[host] - facts.update(dict(tower_host=host, datetime=date_key)) - rec = host_collection.find({"datetime": date_key}) - if rec.count(): - this_fact = rec.next() - this_fact.update(facts) - host_collection.save(this_fact) - else: - host_collection.insert(facts) + + # TODO: in ansible < v2 module_setup is emitted for "smart" fact caching. + # ansible v2 will not emit this message. Thus, this can be removed at that time. + if 'module_setup' in facts_data and len(facts_data) == 1: + return + + try: + host = FactHost.objects.get(hostname=hostname) + except FactHost.DoesNotExist as e: + host = FactHost(hostname=hostname) + host.save() + except FactHost.MultipleObjectsReturned as e: + query = "db['fact_host'].find(hostname=%s)" % hostname + print('Database inconsistent. Multiple FactHost "%s" exist. Try the query %s to find the records.' % (hostname, query)) + return + + (module, facts) = self.process_facts(facts_data) + self.timestamp = datetime.fromtimestamp(date_key, None) + + try: + # Update existing Fact entry + version_obj = FactVersion.objects.get(timestamp=self.timestamp, host=host, module=module) + Fact.objects(id=version_obj.fact.id).update_one(fact=facts) + except FactVersion.DoesNotExist: + # Create new Fact entry + (fact_obj, version_obj) = Fact.add_fact(self.timestamp, facts, host, module) def run_receiver(self): with Socket('fact_cache', 'r') as facts: diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index b7d57212de..9d89df393e 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -16,6 +16,7 @@ from awx.main.models.ad_hoc_commands import * # noqa from awx.main.models.schedules import * # noqa from awx.main.models.activity_stream import * # noqa from awx.main.models.ha import * # noqa +from awx.main.models.fact import * # noqa # Monkeypatch Django serializer to ignore django-taggit fields (which break # the dumpdata command; see https://github.com/alex/django-taggit/issues/155). diff --git a/awx/main/models/fact.py b/awx/main/models/fact.py new file mode 100644 index 0000000000..b97b11d7e3 --- /dev/null +++ b/awx/main/models/fact.py @@ -0,0 +1,49 @@ +from mongoengine import Document, DynamicDocument, DateTimeField, ReferenceField, StringField + +class FactHost(Document): + hostname = StringField(max_length=100, required=True, unique=True) + + # TODO: Consider using hashed index on hostname. django-mongo may not support this but + # executing raw js will + meta = { + 'indexes': [ + 'hostname' + ] + } + +class Fact(DynamicDocument): + timestamp = DateTimeField(required=True) + host = ReferenceField(FactHost, required=True) + module = StringField(max_length=50, required=True) + # fact = + + # TODO: Consider using hashed index on host. django-mongo may not support this but + # executing raw js will + meta = { + 'indexes': [ + '-timestamp', + 'host' + ] + } + + @staticmethod + def add_fact(timestamp, fact, host, module): + fact_obj = Fact(timestamp=timestamp, host=host, module=module, fact=fact) + fact_obj.save() + version_obj = FactVersion(timestamp=timestamp, host=host, module=module, fact=fact_obj) + version_obj.save() + return (fact_obj, version_obj) + +class FactVersion(Document): + timestamp = DateTimeField(required=True) + host = ReferenceField(FactHost, required=True) + module = StringField(max_length=50, required=True) + fact = ReferenceField(Fact, required=True) + # TODO: Consider using hashed index on module. django-mongo may not support this but + # executing raw js will + meta = { + 'indexes': [ + '-timestamp', + 'module' + ] + } diff --git a/awx/main/tests/__init__.py b/awx/main/tests/__init__.py index 6ba54a7802..29d772d495 100644 --- a/awx/main/tests/__init__.py +++ b/awx/main/tests/__init__.py @@ -15,3 +15,6 @@ from awx.main.tests.activity_stream import * # noqa from awx.main.tests.schedules import * # noqa from awx.main.tests.redact import * # noqa from awx.main.tests.views import * # noqa +from awx.main.tests.models import * # noqa +from awx.main.tests.commands import * # noqa +from awx.main.tests.dbtransform import * # noqa diff --git a/awx/main/tests/base.py b/awx/main/tests/base.py index 467e30734e..69fd0e41e2 100644 --- a/awx/main/tests/base.py +++ b/awx/main/tests/base.py @@ -25,6 +25,9 @@ from django.contrib.auth.models import User from django.test.client import Client from django.test.utils import override_settings +# MongoEngine +from mongoengine.connection import get_db + # AWX from awx.main.models import * # noqa from awx.main.backend import LDAPSettings @@ -84,6 +87,11 @@ class BaseTestMixin(QueueTestMixin): def setUp(self): super(BaseTestMixin, self).setUp() + + # Drop mongo database + self.db = get_db() + self.db.connection.drop_database(settings.MONGO_DB) + self.object_ctr = 0 # Save sys.path before tests. self._sys_path = [x for x in sys.path] diff --git a/awx/main/tests/commands/__init__.py b/awx/main/tests/commands/__init__.py new file mode 100644 index 0000000000..0be275c8b8 --- /dev/null +++ b/awx/main/tests/commands/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +from awx.main.tests.commands.run_fact_cache_receiver import * # noqa +from awx.main.tests.commands.commands_monolithic import * # noqa \ No newline at end of file diff --git a/awx/main/tests/commands/base.py b/awx/main/tests/commands/base.py new file mode 100644 index 0000000000..575eb08cf4 --- /dev/null +++ b/awx/main/tests/commands/base.py @@ -0,0 +1,89 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +import StringIO +import sys +import json + +# Django +from django.core.management import call_command + +# AWX +from awx.main.models import * # noqa +from awx.main.tests.base import BaseTestMixin + +class BaseCommandMixin(BaseTestMixin): + ''' + Base class for tests that run management commands. + ''' + + def create_test_inventories(self): + self.setup_users() + self.organizations = self.make_organizations(self.super_django_user, 2) + self.projects = self.make_projects(self.normal_django_user, 2) + self.organizations[0].projects.add(self.projects[1]) + self.organizations[1].projects.add(self.projects[0]) + self.inventories = [] + self.hosts = [] + self.groups = [] + for n, organization in enumerate(self.organizations): + inventory = Inventory.objects.create(name='inventory-%d' % n, + description='description for inventory %d' % n, + organization=organization, + variables=json.dumps({'n': n}) if n else '') + self.inventories.append(inventory) + hosts = [] + for x in xrange(10): + if n > 0: + variables = json.dumps({'ho': 'hum-%d' % x}) + else: + variables = '' + host = inventory.hosts.create(name='host-%02d-%02d.example.com' % (n, x), + inventory=inventory, + variables=variables) + hosts.append(host) + self.hosts.extend(hosts) + groups = [] + for x in xrange(5): + if n > 0: + variables = json.dumps({'gee': 'whiz-%d' % x}) + else: + variables = '' + group = inventory.groups.create(name='group-%d' % x, + inventory=inventory, + variables=variables) + groups.append(group) + group.hosts.add(hosts[x]) + group.hosts.add(hosts[x + 5]) + if n > 0 and x == 4: + group.parents.add(groups[3]) + self.groups.extend(groups) + + def run_command(self, name, *args, **options): + ''' + Run a management command and capture its stdout/stderr along with any + exceptions. + ''' + command_runner = options.pop('command_runner', call_command) + stdin_fileobj = options.pop('stdin_fileobj', None) + options.setdefault('verbosity', 1) + options.setdefault('interactive', False) + original_stdin = sys.stdin + original_stdout = sys.stdout + original_stderr = sys.stderr + if stdin_fileobj: + sys.stdin = stdin_fileobj + sys.stdout = StringIO.StringIO() + sys.stderr = StringIO.StringIO() + result = None + try: + result = command_runner(name, *args, **options) + except Exception as e: + result = e + finally: + captured_stdout = sys.stdout.getvalue() + captured_stderr = sys.stderr.getvalue() + sys.stdin = original_stdin + sys.stdout = original_stdout + sys.stderr = original_stderr + return result, captured_stdout, captured_stderr diff --git a/awx/main/tests/commands.py b/awx/main/tests/commands/commands_monolithic.py similarity index 100% rename from awx/main/tests/commands.py rename to awx/main/tests/commands/commands_monolithic.py diff --git a/awx/main/tests/commands/run_fact_cache_receiver.py b/awx/main/tests/commands/run_fact_cache_receiver.py new file mode 100644 index 0000000000..c55782d015 --- /dev/null +++ b/awx/main/tests/commands/run_fact_cache_receiver.py @@ -0,0 +1,199 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +import time +from datetime import datetime +import mock +import json +import unittest +from copy import deepcopy +from mock import Mock, MagicMock + +# AWX +from awx.main.tests.base import BaseTest +from awx.main.tests.commands.base import BaseCommandMixin +from awx.main.management.commands.run_fact_cache_receiver import FactCacheReceiver, _MODULES +from awx.main.models.fact import * + +__all__ = ['RunFactCacheReceiverUnitTest', 'RunFactCacheReceiverFunctionalTest'] + +TEST_MSG_BASE = { + 'host': 'hostname1', + 'date_key': time.mktime(datetime.utcnow().timetuple()), + 'facts' : { } +} + +TEST_MSG_MODULES = { + 'packages': { + "accountsservice": [ + { + "architecture": "amd64", + "name": "accountsservice", + "source": "apt", + "version": "0.6.35-0ubuntu7.1" + } + ], + "acpid": [ + { + "architecture": "amd64", + "name": "acpid", + "source": "apt", + "version": "1:2.0.21-1ubuntu2" + } + ], + "adduser": [ + { + "architecture": "all", + "name": "adduser", + "source": "apt", + "version": "3.113+nmu3ubuntu3" + } + ], + }, + 'services': [ + { + "name": "acpid", + "source": "sysv", + "state": "running" + }, + { + "name": "apparmor", + "source": "sysv", + "state": "stopped" + }, + { + "name": "atd", + "source": "sysv", + "state": "running" + }, + { + "name": "cron", + "source": "sysv", + "state": "running" + } + ], + 'ansible': { + 'ansible_fact_simple': 'hello world', + 'ansible_fact_complex': { + 'foo': 'bar', + 'hello': [ + 'scooby', + 'dooby', + 'doo' + ] + }, + } +} +# Derived from TEST_MSG_BASE +TEST_MSG = dict(TEST_MSG_BASE) + +def copy_only_module(data, module): + data = deepcopy(data) + data['facts'] = {} + if module == 'ansible': + data['facts'] = deepcopy(TEST_MSG_MODULES[module]) + else: + data['facts'][module] = deepcopy(TEST_MSG_MODULES[module]) + return data + + +class RunFactCacheReceiverFunctionalTest(BaseCommandMixin, BaseTest): + @unittest.skip('''\ +TODO: run_fact_cache_receiver enters a while True loop that never exists. \ +This differs from most other commands that we test for. More logic and work \ +would be required to invoke this case from the command line with little return \ +in terms of increase coverage and confidence.''') + def test_invoke(self): + result, stdout, stderr = self.run_command('run_fact_cache_receiver') + self.assertEqual(result, None) + +class RunFactCacheReceiverUnitTest(BaseTest): + # TODO: Check that timestamp and other attributes are as expected + def check_process_fact_message_module(self, data, module): + fact_found = None + facts = Fact.objects.all() + self.assertEqual(len(facts), 1) + for fact in facts: + if fact.module == module: + fact_found = fact + break + self.assertIsNotNone(fact_found) + #self.assertEqual(data['facts'][module], fact_found[module]) + + fact_found = None + fact_versions = FactVersion.objects.all() + self.assertEqual(len(fact_versions), 1) + for fact in fact_versions: + if fact.module == module: + fact_found = fact + break + self.assertIsNotNone(fact_found) + + + # Ensure that the message flows from the socket through to process_fact_message() + @mock.patch('awx.main.socket.Socket.listen') + def test_run_receiver(self, listen_mock): + listen_mock.return_value = [ TEST_MSG ] + + receiver = FactCacheReceiver() + receiver.process_fact_message = MagicMock(name='process_fact_message') + receiver.run_receiver() + + receiver.process_fact_message.assert_called_once_with(TEST_MSG) + + def test_process_fact_message_ansible(self): + data = copy_only_module(TEST_MSG, 'ansible') + + receiver = FactCacheReceiver() + receiver.process_fact_message(data) + + self.check_process_fact_message_module(data, 'ansible') + + def test_process_fact_message_packages(self): + data = copy_only_module(TEST_MSG, 'packages') + + receiver = FactCacheReceiver() + receiver.process_fact_message(data) + + self.check_process_fact_message_module(data, 'packages') + + def test_process_fact_message_services(self): + data = copy_only_module(TEST_MSG, 'services') + + receiver = FactCacheReceiver() + receiver.process_fact_message(data) + + self.check_process_fact_message_module(data, 'services') + + + # Ensure that only a single host gets created for multiple invocations with the same hostname + def test_process_fact_message_single_host_created(self): + receiver = FactCacheReceiver() + + data = deepcopy(TEST_MSG) + receiver.process_fact_message(data) + data = deepcopy(TEST_MSG) + data['date_key'] = time.mktime(datetime.utcnow().timetuple()) + receiver.process_fact_message(data) + + fact_hosts = FactHost.objects.all() + self.assertEqual(len(fact_hosts), 1) + + def test_process_facts_message_ansible_overwrite(self): + data = copy_only_module(TEST_MSG, 'ansible') + key = 'ansible_overwrite' + value = 'hello world' + + receiver = FactCacheReceiver() + receiver.process_fact_message(data) + + fact = Fact.objects.all()[0] + + data = copy_only_module(TEST_MSG, 'ansible') + data['facts'][key] = value + receiver.process_fact_message(data) + + fact = Fact.objects.get(id=fact.id) + self.assertIn(key, fact.fact) + self.assertEqual(fact.fact[key], value) diff --git a/awx/main/tests/dbtransform.py b/awx/main/tests/dbtransform.py new file mode 100644 index 0000000000..47eb7d75dd --- /dev/null +++ b/awx/main/tests/dbtransform.py @@ -0,0 +1,78 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +from datetime import datetime +from mongoengine.connection import get_db +from mongoengine import connect + +# Django +from django.conf import settings + +# AWX +from awx.main.tests.base import BaseTest +from awx.main.models.fact import * + +__all__ = ['DBTransformTest'] + +class DBTransformTest(BaseTest): + def setUp(self): + super(DBTransformTest, self).setUp() + + # Create a db connection that doesn't have the transformation registered + # Note: this goes through pymongo not mongoengine + self.client = connect(settings.MONGO_DB) + self.db = self.client[settings.MONGO_DB] + + def _create_fact(self): + fact = {} + fact[self.k] = self.v + h = FactHost(hostname='blah') + h.save() + f = Fact(host=h,module='blah',timestamp=datetime.now(),fact=fact) + f.save() + return f + + def create_dot_fact(self): + self.k = 'this.is.a.key' + self.v = 'this.is.a.value' + + self.k_uni = 'this\uff0Eis\uff0Ea\uff0Ekey' + + return self._create_fact() + + def create_dollar_fact(self): + self.k = 'this$is$a$key' + self.v = 'this$is$a$value' + + self.k_uni = 'this\uff04is\uff04a\uff04key' + + return self._create_fact() + + def check_unicode(self, f): + f_raw = self.db.fact.find_one(id=f.id) + self.assertIn(self.k_uni, f_raw['fact']) + self.assertEqual(f_raw['fact'][self.k_uni], self.v) + + # Ensure key . are being transformed to the equivalent unicode into the database + def test_key_transform_dot_unicode_in_storage(self): + f = self.create_dot_fact() + self.check_unicode(f) + + # Ensure key $ are being transformed to the equivalent unicode into the database + def test_key_transform_dollar_unicode_in_storage(self): + f = self.create_dollar_fact() + self.check_unicode(f) + + def check_transform(self): + f = Fact.objects.all()[0] + self.assertIn(self.k, f.fact) + self.assertEqual(f.fact[self.k], self.v) + + def test_key_transform_dot_on_retreive(self): + self.create_dot_fact() + self.check_transform() + + def test_key_transform_dollar_on_retreive(self): + self.create_dollar_fact() + self.check_transform() \ No newline at end of file diff --git a/awx/main/tests/jobs/__init__.py b/awx/main/tests/jobs/__init__.py index 7cba799343..49c40437da 100644 --- a/awx/main/tests/jobs/__init__.py +++ b/awx/main/tests/jobs/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + from awx.main.tests.jobs.jobs_monolithic import * # noqa from survey_password import * # noqa from base import * # noqa diff --git a/awx/main/tests/models/__init__.py b/awx/main/tests/models/__init__.py new file mode 100644 index 0000000000..4ca3023f38 --- /dev/null +++ b/awx/main/tests/models/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +from awx.main.tests.models.fact import * # noqa \ No newline at end of file diff --git a/awx/main/tests/models/fact.py b/awx/main/tests/models/fact.py new file mode 100644 index 0000000000..7414f7456b --- /dev/null +++ b/awx/main/tests/models/fact.py @@ -0,0 +1,87 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +from datetime import datetime + +# Django + +# AWX +from awx.main.models.fact import * +from awx.main.tests.base import BaseTest + +__all__ = ['FactHostTest', 'FactTest'] + +TEST_FACT_DATA = { + 'hostname': 'hostname1', + 'add_fact_data': { + 'timestamp': datetime.now(), + 'host': None, + 'module': 'packages', + 'fact': { + "accountsservice": [ + { + "architecture": "amd64", + "name": "accountsservice", + "source": "apt", + "version": "0.6.35-0ubuntu7.1" + } + ], + "acpid": [ + { + "architecture": "amd64", + "name": "acpid", + "source": "apt", + "version": "1:2.0.21-1ubuntu2" + } + ], + "adduser": [ + { + "architecture": "all", + "name": "adduser", + "source": "apt", + "version": "3.113+nmu3ubuntu3" + } + ], + }, + } +} +# Strip off microseconds because mongo has less precision +TEST_FACT_DATA['add_fact_data']['timestamp'] = TEST_FACT_DATA['add_fact_data']['timestamp'].replace(microsecond=0) + +class FactHostTest(BaseTest): + def test_create_host(self): + host = FactHost(hostname=TEST_FACT_DATA['hostname']) + host.save() + + host = FactHost.objects.get(hostname=TEST_FACT_DATA['hostname']) + self.assertIsNotNone(host, "Host added but not found") + self.assertEqual(TEST_FACT_DATA['hostname'], host.hostname, "Gotten record hostname does not match expected hostname") + + +class FactTest(BaseTest): + def setUp(self): + super(FactTest, self).setUp() + TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save() + + def test_add_fact(self): + (f_obj, v_obj) = Fact.add_fact(**TEST_FACT_DATA['add_fact_data']) + f = Fact.objects.get(id=f_obj.id) + v = FactVersion.objects.get(id=v_obj.id) + + self.assertEqual(f.id, f_obj.id) + self.assertEqual(f.module, TEST_FACT_DATA['add_fact_data']['module']) + self.assertEqual(f.fact, TEST_FACT_DATA['add_fact_data']['fact']) + self.assertEqual(f.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp']) + + # host relationship created + self.assertEqual(f.host.id, TEST_FACT_DATA['add_fact_data']['host'].id) + + # version created and related + self.assertEqual(v.id, v_obj.id) + self.assertEqual(v.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp']) + self.assertEqual(v.host.id, TEST_FACT_DATA['add_fact_data']['host'].id) + self.assertEqual(v.fact.id, f_obj.id) + self.assertEqual(v.fact.module, TEST_FACT_DATA['add_fact_data']['module']) + + diff --git a/awx/plugins/fact_caching/tower.py b/awx/plugins/fact_caching/tower.py index bf6c9211da..f369d9fa79 100755 --- a/awx/plugins/fact_caching/tower.py +++ b/awx/plugins/fact_caching/tower.py @@ -32,6 +32,8 @@ import sys import time import datetime +import json +from copy import deepcopy from ansible import constants as C from ansible.cache.base import BaseCacheModule @@ -47,6 +49,7 @@ class CacheModule(BaseCacheModule): # Basic in-memory caching for typical runs self._cache = {} + self._cache_prev = {} # This is the local tower zmq connection self._tower_connection = C.CACHE_PLUGIN_CONNECTION @@ -54,20 +57,67 @@ class CacheModule(BaseCacheModule): try: self.context = zmq.Context() self.socket = self.context.socket(zmq.REQ) + self.socket.setsockopt(zmq.RCVTIMEO, 4000) + self.socket.setsockopt(zmq.LINGER, 2000) self.socket.connect(self._tower_connection) except Exception, e: print("Connection to zeromq failed at %s with error: %s" % (str(self._tower_connection), str(e))) sys.exit(1) + def identify_ansible_facts(self, facts): + ansible_keys = {} + for k in facts.keys(): + if k.startswith('ansible_'): + ansible_keys[k] = 1 + return ansible_keys + + def identify_new_module(self, key, value): + if key in self._cache_prev: + value_old = self._cache_prev[key] + for k,v in value.iteritems(): + if k not in value_old: + if not k.startswith('ansible_'): + return k + return None + def get(self, key): return self._cache.get(key) + ''' + get() returns a reference to the fact object (usually a dict). The object is modified directly, + then set is called. Effectively, pre-determining the set logic. + + The below logic creates a backup of the cache each set. The values are now preserved across set() calls. + + For a given key. The previous value is looked at for new keys that aren't of the form 'ansible_'. + If found, send the value of the found key. + If not found, send all the key value pairs of the form 'ansible_' (we presume set() is called because + of an ansible fact module invocation) + + More simply stated... + In value, if a new key is found at the top most dict then consider this a module request and only + emit the facts for the found top-level key. + + If a new key is not found, assume set() was called as a result of ansible facts scan. Thus, emit + all facts of the form 'ansible_'. + ''' def set(self, key, value): + module = self.identify_new_module(key, value) + # Assume ansible fact triggered the set if no new module found + facts = {} + if not module: + keys = self.identify_ansible_facts(value) + for k in keys: + facts[k] = value[k] + else: + facts[module] = value[module] + + self._cache_prev = deepcopy(self._cache) self._cache[key] = value # Emit fact data to tower for processing - self.socket.send_json(dict(host=key, facts=value, date_key=self.date_key)) + self.socket.send_json(dict(host=key, facts=facts, date_key=self.date_key)) self.socket.recv() def keys(self): diff --git a/awx/settings/__init__.py b/awx/settings/__init__.py index 893555cc13..73f421a6e1 100644 --- a/awx/settings/__init__.py +++ b/awx/settings/__init__.py @@ -1,2 +1,10 @@ # Copyright (c) 2014 AnsibleWorks, Inc. # All Rights Reserved. + +from django.conf import settings +from mongoengine import connect +from mongoengine.connection import get_db +from awx.main.dbtransform import register_key_transform + +connect(settings.MONGO_DB) +register_key_transform(get_db()) \ No newline at end of file diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index adf6cfd7a7..d37b047ff2 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -7,6 +7,8 @@ import glob from datetime import timedelta import tempfile +MONGO_DB = 'system_tracking' + # Update this module's local settings from the global settings module. from django.conf import global_settings this_module = sys.modules[__name__] diff --git a/awx/settings/development.py b/awx/settings/development.py index fa9e067744..d365b25f57 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -14,6 +14,8 @@ from split_settings.tools import optional, include # Load default settings. from defaults import * +MONGO_DB = 'system_tracking_dev' + # Disable capturing all SQL queries when running celeryd in development. if 'celeryd' in sys.argv: SQL_DEBUG = False diff --git a/requirements/dev_local.txt b/requirements/dev_local.txt index 7579a37993..c1c6c8420d 100644 --- a/requirements/dev_local.txt +++ b/requirements/dev_local.txt @@ -55,7 +55,6 @@ mongo-python-driver-2.8.tar.gz # Needed by pyrax: #httplib2-0.8.tar.gz #keyring-3.7.zip - #mock-1.0.1.tar.gz #python-swiftclient-2.0.3.tar.gz #rackspace-novaclient-1.4.tar.gz # Remaining dev/prod packages: @@ -78,6 +77,8 @@ mongo-python-driver-2.8.tar.gz #mongoengine-0.9.0.tar.gz # Dev-only packages: +# Needed for tests +mock-1.0.1.tar.gz # Needed by django-debug-toolbar: sqlparse-0.1.11.tar.gz # Needed for Python2.6 support: