diff --git a/awx/fact/models/fact.py b/awx/fact/models/fact.py index 63bb08dfcc..e3705ee493 100644 --- a/awx/fact/models/fact.py +++ b/awx/fact/models/fact.py @@ -1,7 +1,24 @@ # Copyright (c) 2015 Ansible, Inc. # All Rights Reserved -from mongoengine import Document, DynamicDocument, DateTimeField, ReferenceField, StringField +from mongoengine.base import BaseField +from mongoengine import Document, DateTimeField, ReferenceField, StringField +from awx.fact.utils.dbtransform import KeyTransform + +key_transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')]) + +class TransformField(BaseField): + def to_python(self, value): + return key_transform.transform_outgoing(value, None) + + def prepare_query_value(self, op, value): + if op == 'set': + value = key_transform.transform_incoming(value, None) + return super(TransformField, self).prepare_query_value(op, value) + + def to_mongo(self, value): + value = key_transform.transform_incoming(value, None) + return value class FactHost(Document): hostname = StringField(max_length=100, required=True, unique=True) @@ -21,11 +38,11 @@ class FactHost(Document): return host.id return None -class Fact(DynamicDocument): +class Fact(Document): timestamp = DateTimeField(required=True) host = ReferenceField(FactHost, required=True) module = StringField(max_length=50, required=True) - # fact = + fact = TransformField(required=True) # TODO: Consider using hashed index on host. django-mongo may not support this but # executing raw js will diff --git a/awx/fact/tests/models/fact/__init__.py b/awx/fact/tests/models/fact/__init__.py index 151002610f..bdb0d01136 100644 --- a/awx/fact/tests/models/fact/__init__.py +++ b/awx/fact/tests/models/fact/__init__.py @@ -4,4 +4,6 @@ from __future__ import absolute_import from .fact_simple import * # noqa +from .fact_transform_pymongo import * # noqa +from .fact_transform import * # noqa from .fact_get_single_facts import * # noqa diff --git a/awx/fact/tests/models/fact/fact_transform.py b/awx/fact/tests/models/fact/fact_transform.py new file mode 100644 index 0000000000..6661f81179 --- /dev/null +++ b/awx/fact/tests/models/fact/fact_transform.py @@ -0,0 +1,112 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +from __future__ import absolute_import +from datetime import datetime + +# Django +from django.conf import settings + +# Pymongo +import pymongo + +# AWX +from awx.fact.models.fact import * # noqa +from .base import BaseFactTest + +__all__ = ['FactTransformTest', 'FactTransformUpdateTest',] + +TEST_FACT_DATA = { + 'hostname': 'hostname1', + 'add_fact_data': { + 'timestamp': datetime.now(), + 'host': None, + 'module': 'packages', + 'fact': { + "acpid3.4": [ + { + "version": "1:2.0.21-1ubuntu2", + "deeper.key": "some_value" + } + ], + "adduser.2": [ + { + "source": "apt", + "version": "3.113+nmu3ubuntu3" + } + ], + "what.ever." : { + "shallowish.key": "some_shallow_value" + } + }, + } +} +# Strip off microseconds because mongo has less precision +BaseFactTest.normalize_timestamp(TEST_FACT_DATA) + +class FactTransformTest(BaseFactTest): + def setUp(self): + super(FactTransformTest, self).setUp() + # TODO: get host settings from config + self.client = pymongo.MongoClient('localhost', 27017) + self.db2 = self.client[settings.MONGO_DB] + + self.create_host_document(TEST_FACT_DATA) + + def setup_create_fact_dot(self): + self.data = TEST_FACT_DATA + self.f = Fact(**TEST_FACT_DATA['add_fact_data']) + self.f.save() + + def setup_create_fact_dollar(self): + self.data = TEST_FACT_DATA + self.f = Fact(**TEST_FACT_DATA['add_fact_data']) + self.f.save() + + def test_fact_with_dot_serialized(self): + self.setup_create_fact_dot() + + q = { + '_id': self.f.id + } + + # Bypass mongoengine and pymongo transform to get record + f_dict = self.db2['fact'].find_one(q) + self.assertIn('acpid3\uff0E4', f_dict['fact']) + + def test_fact_with_dot_serialized_pymongo(self): + #self.setup_create_fact_dot() + + f = self.db['fact'].insert({ + 'hostname': TEST_FACT_DATA['hostname'], + 'fact': TEST_FACT_DATA['add_fact_data']['fact'], + 'timestamp': TEST_FACT_DATA['add_fact_data']['timestamp'], + 'host': TEST_FACT_DATA['add_fact_data']['host'].id, + 'module': TEST_FACT_DATA['add_fact_data']['module'] + }) + + q = { + '_id': f + } + # Bypass mongoengine and pymongo transform to get record + f_dict = self.db2['fact'].find_one(q) + self.assertIn('acpid3\uff0E4', f_dict['fact']) + + def test_fact_with_dot_deserialized_pymongo(self): + self.setup_create_fact_dot() + + q = { + '_id': self.f.id + } + f_dict = self.db['fact'].find_one(q) + self.assertIn('acpid3.4', f_dict['fact']) + + def test_fact_with_dot_deserialized(self): + self.setup_create_fact_dot() + + f = Fact.objects.get(id=self.f.id) + self.assertIn('acpid3.4', f.fact) + +class FactTransformUpdateTest(BaseFactTest): + pass diff --git a/awx/fact/tests/models/fact/fact_transform_pymongo.py b/awx/fact/tests/models/fact/fact_transform_pymongo.py new file mode 100644 index 0000000000..7cf81e4650 --- /dev/null +++ b/awx/fact/tests/models/fact/fact_transform_pymongo.py @@ -0,0 +1,96 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +from __future__ import absolute_import +from datetime import datetime + +# Django +from django.conf import settings + +# Pymongo +import pymongo + +# AWX +from awx.fact.models.fact import * # noqa +from .base import BaseFactTest + +__all__ = ['FactSerializePymongoTest', 'FactDeserializePymongoTest',] + +class FactPymongoBaseTest(BaseFactTest): + def setUp(self): + super(FactPymongoBaseTest, self).setUp() + # TODO: get host settings from config + self.client = pymongo.MongoClient('localhost', 27017) + self.db2 = self.client[settings.MONGO_DB] + + def _create_fact(self): + fact = {} + fact[self.k] = self.v + q = { + 'hostname': 'blah' + } + h = self.db['fact_host'].insert(q) + q = { + 'host': h, + 'module': 'blah', + 'timestamp': datetime.now(), + 'fact': fact + } + f = self.db['fact'].insert(q) + return f + + def check_transform(self, id): + raise RuntimeError("Must override") + + def create_dot_fact(self): + self.k = 'this.is.a.key' + self.v = 'this.is.a.value' + + self.k_uni = 'this\uff0Eis\uff0Ea\uff0Ekey' + + return self._create_fact() + + def create_dollar_fact(self): + self.k = 'this$is$a$key' + self.v = 'this$is$a$value' + + self.k_uni = 'this\uff04is\uff04a\uff04key' + + return self._create_fact() + +class FactSerializePymongoTest(FactPymongoBaseTest): + def check_transform(self, id): + q = { + '_id': id + } + f = self.db2.fact.find_one(q) + self.assertIn(self.k_uni, f['fact']) + self.assertEqual(f['fact'][self.k_uni], self.v) + + # Ensure key . are being transformed to the equivalent unicode into the database + def test_key_transform_dot(self): + f = self.create_dot_fact() + self.check_transform(f) + + # Ensure key $ are being transformed to the equivalent unicode into the database + def test_key_transform_dollar(self): + f = self.create_dollar_fact() + self.check_transform(f) + +class FactDeserializePymongoTest(FactPymongoBaseTest): + def check_transform(self, id): + q = { + '_id': id + } + f = self.db.fact.find_one(q) + self.assertIn(self.k, f['fact']) + self.assertEqual(f['fact'][self.k], self.v) + + def test_key_transform_dot(self): + f = self.create_dot_fact() + self.check_transform(f) + + def test_key_transform_dollar(self): + f = self.create_dollar_fact() + self.check_transform(f) diff --git a/awx/fact/tests/utils/dbtransform.py b/awx/fact/tests/utils/dbtransform.py index a11375223d..97bfad9c65 100644 --- a/awx/fact/tests/utils/dbtransform.py +++ b/awx/fact/tests/utils/dbtransform.py @@ -1,77 +1,112 @@ # Copyright (c) 2015 Ansible, Inc. # All Rights Reserved -# Python -from datetime import datetime -from mongoengine import connect - -# Django -from django.conf import settings - # AWX -from awx.main.tests.base import BaseTest, MongoDBRequired +from awx.main.tests.base import BaseTest from awx.fact.models.fact import * # noqa +from awx.fact.utils.dbtransform import KeyTransform -__all__ = ['DBTransformTest'] +#__all__ = ['DBTransformTest', 'KeyTransformUnitTest'] +__all__ = ['KeyTransformUnitTest'] -class DBTransformTest(BaseTest, MongoDBRequired): +class KeyTransformUnitTest(BaseTest): def setUp(self): - super(DBTransformTest, self).setUp() + super(KeyTransformUnitTest, self).setUp() + self.key_transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')]) - # Create a db connection that doesn't have the transformation registered - # Note: this goes through pymongo not mongoengine - self.client = connect(settings.MONGO_DB) - self.db = self.client[settings.MONGO_DB] + def test_no_replace(self): + value = { + "a_key_with_a_dict" : { + "key" : "value", + "nested_key_with_dict": { + "nested_key_with_value" : "deep_value" + } + } + } - def _create_fact(self): - fact = {} - fact[self.k] = self.v - h = FactHost(hostname='blah') - h.save() - f = Fact(host=h,module='blah',timestamp=datetime.now(),fact=fact) - f.save() - return f + data = self.key_transform.transform_incoming(value, None) + self.assertEqual(data, value) - def create_dot_fact(self): - self.k = 'this.is.a.key' - self.v = 'this.is.a.value' + data = self.key_transform.transform_outgoing(value, None) + self.assertEqual(data, value) - self.k_uni = 'this\uff0Eis\uff0Ea\uff0Ekey' + def test_complex(self): + value = { + "a.key.with.a.dict" : { + "key" : "value", + "nested.key.with.dict": { + "nested.key.with.value" : "deep_value" + } + } + } + value_transformed = { + "a\uff0Ekey\uff0Ewith\uff0Ea\uff0Edict" : { + "key" : "value", + "nested\uff0Ekey\uff0Ewith\uff0Edict": { + "nested\uff0Ekey\uff0Ewith\uff0Evalue" : "deep_value" + } + } + } - return self._create_fact() + data = self.key_transform.transform_incoming(value, None) + self.assertEqual(data, value_transformed) - def create_dollar_fact(self): - self.k = 'this$is$a$key' - self.v = 'this$is$a$value' + data = self.key_transform.transform_outgoing(value_transformed, None) + self.assertEqual(data, value) - self.k_uni = 'this\uff04is\uff04a\uff04key' + def test_simple(self): + value = { + "a.key" : "value" + } + value_transformed = { + "a\uff0Ekey" : "value" + } - return self._create_fact() + data = self.key_transform.transform_incoming(value, None) + self.assertEqual(data, value_transformed) - def check_unicode(self, f): - f_raw = self.db.fact.find_one(id=f.id) - self.assertIn(self.k_uni, f_raw['fact']) - self.assertEqual(f_raw['fact'][self.k_uni], self.v) + data = self.key_transform.transform_outgoing(value_transformed, None) + self.assertEqual(data, value) - # Ensure key . are being transformed to the equivalent unicode into the database - def test_key_transform_dot_unicode_in_storage(self): - f = self.create_dot_fact() - self.check_unicode(f) + def test_nested_dict(self): + value = { + "a.key.with.a.dict" : { + "nested.key." : "value" + } + } + value_transformed = { + "a\uff0Ekey\uff0Ewith\uff0Ea\uff0Edict" : { + "nested\uff0Ekey\uff0E" : "value" + } + } - # Ensure key $ are being transformed to the equivalent unicode into the database - def test_key_transform_dollar_unicode_in_storage(self): - f = self.create_dollar_fact() - self.check_unicode(f) + data = self.key_transform.transform_incoming(value, None) + self.assertEqual(data, value_transformed) + + data = self.key_transform.transform_outgoing(value_transformed, None) + self.assertEqual(data, value) - def check_transform(self): - f = Fact.objects.all()[0] - self.assertIn(self.k, f.fact) - self.assertEqual(f.fact[self.k], self.v) + def test_array(self): + value = { + "a.key.with.an.array" : [ + { + "key.with.dot" : "value" + } + ] + } + value_transformed = { + "a\uff0Ekey\uff0Ewith\uff0Ean\uff0Earray" : [ + { + "key\uff0Ewith\uff0Edot" : "value" + } + ] + } + data = self.key_transform.transform_incoming(value, None) + self.assertEqual(data, value_transformed) + + data = self.key_transform.transform_outgoing(value_transformed, None) + self.assertEqual(data, value) - def test_key_transform_dot_on_retreive(self): - self.create_dot_fact() - self.check_transform() - - def test_key_transform_dollar_on_retreive(self): - self.create_dollar_fact() - self.check_transform() +''' +class DBTransformTest(BaseTest, MongoDBRequired): +''' diff --git a/awx/fact/utils/dbtransform.py b/awx/fact/utils/dbtransform.py index f88708a467..98ce8180c9 100644 --- a/awx/fact/utils/dbtransform.py +++ b/awx/fact/utils/dbtransform.py @@ -1,55 +1,55 @@ # Copyright (c) 2014, Ansible, Inc. # All Rights Reserved. + +# Pymongo from pymongo.son_manipulator import SONManipulator -''' -Inspired by: https://stackoverflow.com/questions/8429318/how-to-use-dot-in-field-name/20698802#20698802 - -Replace . and $ with unicode values -''' class KeyTransform(SONManipulator): + def __init__(self, replace): self.replace = replace - def transform_key(self, key, replace, replacement): - """Transform key for saving to database.""" - return key.replace(replace, replacement) + def replace_key(self, key): + for (replace, replacement) in self.replace: + key = key.replace(replace, replacement) + return key - def revert_key(self, key, replace, replacement): - """Restore transformed key returning from database.""" - return key.replace(replacement, replace) + def revert_key(self, key): + for (replacement, replace) in self.replace: + key = key.replace(replace, replacement) + return key + + def replace_incoming(self, obj): + if isinstance(obj, dict): + value = {} + for k, v in obj.items(): + value[self.replace_key(k)] = self.replace_incoming(v) + elif isinstance(obj, list): + value = [self.replace_incoming(elem) + for elem in obj] + else: + value = obj + + return value + + def replace_outgoing(self, obj): + if isinstance(obj, dict): + value = {} + for k, v in obj.items(): + value[self.revert_key(k)] = self.replace_outgoing(v) + elif isinstance(obj, list): + value = [self.replace_outgoing(elem) + for elem in obj] + else: + value = obj + + return value def transform_incoming(self, son, collection): - """Recursively replace all keys that need transforming.""" - for (key, value) in son.items(): - for r in self.replace: - replace = r[0] - replacement = r[1] - if replace in key: - if isinstance(value, dict): - son[self.transform_key(key, replace, replacement)] = self.transform_incoming( - son.pop(key), collection) - else: - son[self.transform_key(key, replace, replacement)] = son.pop(key) - elif isinstance(value, dict): # recurse into sub-docs - son[key] = self.transform_incoming(value, collection) - return son + return self.replace_incoming(son) def transform_outgoing(self, son, collection): - """Recursively restore all transformed keys.""" - for (key, value) in son.items(): - for r in self.replace: - replace = r[0] - replacement = r[1] - if replacement in key: - if isinstance(value, dict): - son[self.revert_key(key, replace, replacement)] = self.transform_outgoing( - son.pop(key), collection) - else: - son[self.revert_key(key, replace, replacement)] = son.pop(key) - elif isinstance(value, dict): # recurse into sub-docs - son[key] = self.transform_outgoing(value, collection) - return son + return self.replace_outgoing(son) def register_key_transform(db): db.add_son_manipulator(KeyTransform([('.', '\uff0E'), ('$', '\uff04')])) diff --git a/awx/main/tests/commands/run_fact_cache_receiver.py b/awx/main/tests/commands/run_fact_cache_receiver.py index 1a63b25180..d0f2d17b49 100644 --- a/awx/main/tests/commands/run_fact_cache_receiver.py +++ b/awx/main/tests/commands/run_fact_cache_receiver.py @@ -87,6 +87,8 @@ TEST_MSG_MODULES = { # Derived from TEST_MSG_BASE TEST_MSG = dict(TEST_MSG_BASE) +TEST_MSG_LARGE = {u'ansible_product_version': u'To Be Filled By O.E.M.', u'ansible_memory_mb': {u'real': {u'total': 32062, u'used': 8079, u'free': 23983}, u'swap': {u'cached': 0, u'total': 0, u'used': 0, u'free': 0}, u'nocache': {u'used': 4339, u'free': 27723}}, u'ansible_user_dir': u'/root', u'ansible_userspace_bits': u'64', u'ansible_distribution_version': u'14.04', u'ansible_virtualization_role': u'guest', u'ansible_env': {u'ANSIBLE_PARAMIKO_RECORD_HOST_KEYS': u'False', u'LC_CTYPE': u'en_US.UTF-8', u'JOB_CALLBACK_DEBUG': u'1', u'_MP_FORK_LOGFILE_': u'', u'HOME': u'/', u'REST_API_TOKEN': u'122-5deb0d6fcec85f3bf44fec6ce170600c', u'LANG': u'en_US.UTF-8', u'SHELL': u'/bin/bash', u'_MP_FORK_LOGFORMAT_': u'[%(asctime)s: %(levelname)s/%(processName)s] %(message)s', u'_': u'/usr/bin/make', u'DJANGO_PROJECT_DIR': u'/tower_devel', u'MFLAGS': u'-w', u'JOB_ID': u'122', u'PYTHONPATH': u'/tower_devel/awx/lib/site-packages:', u'_MP_FORK_LOGLEVEL_': u'10', u'ANSIBLE_CACHE_PLUGIN_CONNECTION': u'tcp://127.0.0.1:6564', u'ANSIBLE_LIBRARY': u'/tower_devel/awx/plugins/library', u'CELERY_LOG_LEVEL': u'10', u'HOSTNAME': u'2842b3619fa8', u'MAKELEVEL': u'2', u'TMUX_PANE': u'%1', u'DJANGO_LIVE_TEST_SERVER_ADDRESS': u'localhost:9013-9199', u'CELERY_LOG_REDIRECT': u'1', u'PATH': u'/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', u'CALLBACK_CONSUMER_PORT': u'tcp://127.0.0.1:5557', u'MAKEFLAGS': u'w', u'ANSIBLE_CALLBACK_PLUGINS': u'/tower_devel/awx/plugins/callback', u'TERM': u'screen', u'TZ': u'America/New_York', u'LANGUAGE': u'en_US:en', u'ANSIBLE_SSH_CONTROL_PATH': u'/tmp/ansible_tower_y3xGdA/cp/ansible-ssh-%%h-%%p-%%r', u'SHLVL': u'1', u'CELERY_LOG_FILE': u'', u'ANSIBLE_HOST_KEY_CHECKING': u'False', u'TMUX': u'/tmp/tmux-0/default,3719,0', u'CELERY_LOADER': u'djcelery.loaders.DjangoLoader', u'LC_ALL': u'en_US.UTF-8', u'ANSIBLE_FORCE_COLOR': u'True', u'REST_API_URL': u'http://127.0.0.1:8013', u'CELERY_LOG_REDIRECT_LEVEL': u'WARNING', u'INVENTORY_HOSTVARS': u'True', u'ANSIBLE_CACHE_PLUGIN': u'tower', u'INVENTORY_ID': u'1', u'PWD': u'/tower_devel/awx/playbooks', u'DJANGO_SETTINGS_MODULE': u'awx.settings.development', u'ANSIBLE_CACHE_PLUGINS': u'/tower_devel/awx/plugins/fact_caching'}, u'ansible_lo': {u'mtu': 65536, u'device': u'lo', u'promisc': False, u'ipv4': {u'netmask': u'255.0.0.0', u'network': u'127.0.0.0', u'address': u'127.0.0.1'}, u'ipv6': [{u'scope': u'host', u'prefix': u'128', u'address': u'::1'}], u'active': True, u'type': u'loopback'}, u'ansible_memtotal_mb': 32062, u'ansible_architecture': u'x86_64', u'ansible_default_ipv4': {u'alias': u'eth0', u'netmask': u'255.255.0.0', u'macaddress': u'02:42:ac:11:00:01', u'network': u'172.17.0.0', u'address': u'172.17.0.1', u'interface': u'eth0', u'type': u'ether', u'gateway': u'172.17.42.1', u'mtu': 1500}, u'ansible_swapfree_mb': 0, u'ansible_default_ipv6': {}, u'ansible_cmdline': {u'nomodeset': True, u'rw': True, u'initrd': u'EFIarchinitramfs-arch.img', u'rootfstype': u'ext4', u'root': u'/dev/sda4', u'systemd.unit': u'graphical.target'}, u'ansible_selinux': False, u'ansible_userspace_architecture': u'x86_64', u'ansible_product_uuid': u'00020003-0004-0005-0006-000700080009', u'ansible_pkg_mgr': u'apt', u'ansible_memfree_mb': 23983, u'ansible_distribution': u'Ubuntu', u'ansible_processor_count': 1, u'ansible_hostname': u'2842b3619fa8', u'ansible_all_ipv6_addresses': [u'fe80::42:acff:fe11:1'], u'ansible_interfaces': [u'lo', u'eth0'], u'ansible_kernel': u'4.0.1-1-ARCH', u'ansible_fqdn': u'2842b3619fa8', u'ansible_mounts': [{u'uuid': u'NA', u'size_total': 10434699264, u'mount': u'/', u'size_available': 4918865920, u'fstype': u'ext4', u'device': u'/dev/mapper/docker-8:4-18219321-2842b3619fa885d19e47302009754a4bfd54c1b32c7f21e98f38c7fe7412d3d0', u'options': u'rw,relatime,discard,stripe=16,data=ordered'}, {u'uuid': u'NA', u'size_total': 570629263360, u'mount': u'/tower_devel', u'size_available': 240166572032, u'fstype': u'ext4', u'device': u'/dev/sda4', u'options': u'rw,relatime,data=ordered'}, {u'uuid': u'NA', u'size_total': 570629263360, u'mount': u'/etc/resolv.conf', u'size_available': 240166572032, u'fstype': u'ext4', u'device': u'/dev/sda4', u'options': u'rw,relatime,data=ordered'}, {u'uuid': u'NA', u'size_total': 570629263360, u'mount': u'/etc/hostname', u'size_available': 240166572032, u'fstype': u'ext4', u'device': u'/dev/sda4', u'options': u'rw,relatime,data=ordered'}, {u'uuid': u'NA', u'size_total': 570629263360, u'mount': u'/etc/hosts', u'size_available': 240166572032, u'fstype': u'ext4', u'device': u'/dev/sda4', u'options': u'rw,relatime,data=ordered'}], u'ansible_user_shell': u'/bin/bash', u'ansible_nodename': u'2842b3619fa8', u'ansible_product_serial': u'To Be Filled By O.E.M.', u'ansible_form_factor': u'Desktop', u'ansible_fips': False, u'ansible_user_id': u'root', u'ansible_domain': u'', u'ansible_date_time': {u'month': u'05', u'second': u'47', u'iso8601_micro': u'2015-05-01T19:46:47.868456Z', u'year': u'2015', u'date': u'2015-05-01', u'iso8601': u'2015-05-01T19:46:47Z', u'day': u'01', u'minute': u'46', u'tz': u'EDT', u'hour': u'15', u'tz_offset': u'-0400', u'epoch': u'1430509607', u'weekday': u'Friday', u'time': u'15:46:47'}, u'ansible_processor_cores': 4, u'ansible_processor_vcpus': 4, u'ansible_bios_version': u'P1.80', u'ansible_processor': [u'GenuineIntel', u'Intel(R) Core(TM) i5-2310 CPU @ 2.90GHz', u'GenuineIntel', u'Intel(R) Core(TM) i5-2310 CPU @ 2.90GHz', u'GenuineIntel', u'Intel(R) Core(TM) i5-2310 CPU @ 2.90GHz', u'GenuineIntel', u'Intel(R) Core(TM) i5-2310 CPU @ 2.90GHz'], u'ansible_virtualization_type': u'docker', u'ansible_distribution_release': u'trusty', u'ansible_system_vendor': u'To Be Filled By O.E.M.', u'ansible_os_family': u'Debian', u'ansible_user_gid': 0, u'ansible_swaptotal_mb': 0, u'ansible_system': u'Linux', u'ansible_devices': {u'sda': {u'sectorsize': u'4096', u'vendor': u'ATA', u'host': u'', u'support_discard': u'0', u'model': u'ST1000DM003-9YN1', u'size': u'7.28 TB', u'scheduler_mode': u'cfq', u'rotational': u'1', u'sectors': u'1953525168', u'removable': u'0', u'holders': [], u'partitions': {u'sda4': {u'start': u'820979712', u'sectorsize': 512, u'sectors': u'1132545423', u'size': u'540.04 GB'}, u'sda2': {u'start': u'206848', u'sectorsize': 512, u'sectors': u'262144', u'size': u'128.00 MB'}, u'sda3': {u'start': u'468992', u'sectorsize': 512, u'sectors': u'820510720', u'size': u'391.25 GB'}, u'sda1': {u'start': u'2048', u'sectorsize': 512, u'sectors': u'204800', u'size': u'100.00 MB'}}}}, u'ansible_user_uid': 0, u'ansible_distribution_major_version': u'14', u'ansible_lsb': {u'major_release': u'14', u'release': u'14.04', u'codename': u'trusty', u'description': u'Ubuntu 14.04.1 LTS', u'id': u'Ubuntu'}, u'ansible_bios_date': u'12/05/2012', u'ansible_machine': u'x86_64', u'ansible_user_gecos': u'root', u'ansible_processor_threads_per_core': 1, u'ansible_eth0': {u'device': u'eth0', u'promisc': False, u'macaddress': u'02:42:ac:11:00:01', u'ipv4': {u'netmask': u'255.255.0.0', u'network': u'172.17.0.0', u'address': u'172.17.0.1'}, u'ipv6': [{u'scope': u'link', u'prefix': u'64', u'address': u'fe80::42:acff:fe11:1'}], u'active': True, u'type': u'ether', u'mtu': 1500}, u'ansible_product_name': u'To Be Filled By O.E.M.', u'ansible_all_ipv4_addresses': [u'172.17.0.1'], u'ansible_python_version': u'2.7.6'} # noqa + def copy_only_module(data, module): data = deepcopy(data) data['facts'] = {} @@ -182,7 +184,7 @@ class RunFactCacheReceiverUnitTest(BaseTest, MongoDBRequired): def test_process_facts_message_ansible_overwrite(self): data = copy_only_module(TEST_MSG, 'ansible') - key = 'ansible_overwrite' + key = 'ansible.overwrite' value = 'hello world' receiver = FactCacheReceiver() @@ -197,3 +199,21 @@ class RunFactCacheReceiverUnitTest(BaseTest, MongoDBRequired): fact = Fact.objects.get(id=fact.id) self.assertIn(key, fact.fact) self.assertEqual(fact.fact[key], value) + self.assertEqual(fact.fact, data['facts']) + + def test_large_overwrite(self): + data = deepcopy(TEST_MSG_BASE) + data['facts'] = { + 'ansible': {} + } + + receiver = FactCacheReceiver() + receiver.process_fact_message(data) + + fact = Fact.objects.all()[0] + + data['facts']['ansible'] = TEST_MSG_LARGE + receiver.process_fact_message(data) + + fact = Fact.objects.get(id=fact.id) + self.assertEqual(fact.fact, data['facts']['ansible'])