diff --git a/awx/fact/models/fact.py b/awx/fact/models/fact.py index 63bb08dfcc..e3705ee493 100644 --- a/awx/fact/models/fact.py +++ b/awx/fact/models/fact.py @@ -1,7 +1,24 @@ # Copyright (c) 2015 Ansible, Inc. # All Rights Reserved -from mongoengine import Document, DynamicDocument, DateTimeField, ReferenceField, StringField +from mongoengine.base import BaseField +from mongoengine import Document, DateTimeField, ReferenceField, StringField +from awx.fact.utils.dbtransform import KeyTransform + +key_transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')]) + +class TransformField(BaseField): + def to_python(self, value): + return key_transform.transform_outgoing(value, None) + + def prepare_query_value(self, op, value): + if op == 'set': + value = key_transform.transform_incoming(value, None) + return super(TransformField, self).prepare_query_value(op, value) + + def to_mongo(self, value): + value = key_transform.transform_incoming(value, None) + return value class FactHost(Document): hostname = StringField(max_length=100, required=True, unique=True) @@ -21,11 +38,11 @@ class FactHost(Document): return host.id return None -class Fact(DynamicDocument): +class Fact(Document): timestamp = DateTimeField(required=True) host = ReferenceField(FactHost, required=True) module = StringField(max_length=50, required=True) - # fact = + fact = TransformField(required=True) # TODO: Consider using hashed index on host. django-mongo may not support this but # executing raw js will diff --git a/awx/fact/tests/models/fact/__init__.py b/awx/fact/tests/models/fact/__init__.py index 151002610f..bdb0d01136 100644 --- a/awx/fact/tests/models/fact/__init__.py +++ b/awx/fact/tests/models/fact/__init__.py @@ -4,4 +4,6 @@ from __future__ import absolute_import from .fact_simple import * # noqa +from .fact_transform_pymongo import * # noqa +from .fact_transform import * # noqa from .fact_get_single_facts import * # noqa diff --git a/awx/fact/tests/models/fact/fact_transform.py b/awx/fact/tests/models/fact/fact_transform.py new file mode 100644 index 0000000000..6661f81179 --- /dev/null +++ b/awx/fact/tests/models/fact/fact_transform.py @@ -0,0 +1,112 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +from __future__ import absolute_import +from datetime import datetime + +# Django +from django.conf import settings + +# Pymongo +import pymongo + +# AWX +from awx.fact.models.fact import * # noqa +from .base import BaseFactTest + +__all__ = ['FactTransformTest', 'FactTransformUpdateTest',] + +TEST_FACT_DATA = { + 'hostname': 'hostname1', + 'add_fact_data': { + 'timestamp': datetime.now(), + 'host': None, + 'module': 'packages', + 'fact': { + "acpid3.4": [ + { + "version": "1:2.0.21-1ubuntu2", + "deeper.key": "some_value" + } + ], + "adduser.2": [ + { + "source": "apt", + "version": "3.113+nmu3ubuntu3" + } + ], + "what.ever." : { + "shallowish.key": "some_shallow_value" + } + }, + } +} +# Strip off microseconds because mongo has less precision +BaseFactTest.normalize_timestamp(TEST_FACT_DATA) + +class FactTransformTest(BaseFactTest): + def setUp(self): + super(FactTransformTest, self).setUp() + # TODO: get host settings from config + self.client = pymongo.MongoClient('localhost', 27017) + self.db2 = self.client[settings.MONGO_DB] + + self.create_host_document(TEST_FACT_DATA) + + def setup_create_fact_dot(self): + self.data = TEST_FACT_DATA + self.f = Fact(**TEST_FACT_DATA['add_fact_data']) + self.f.save() + + def setup_create_fact_dollar(self): + self.data = TEST_FACT_DATA + self.f = Fact(**TEST_FACT_DATA['add_fact_data']) + self.f.save() + + def test_fact_with_dot_serialized(self): + self.setup_create_fact_dot() + + q = { + '_id': self.f.id + } + + # Bypass mongoengine and pymongo transform to get record + f_dict = self.db2['fact'].find_one(q) + self.assertIn('acpid3\uff0E4', f_dict['fact']) + + def test_fact_with_dot_serialized_pymongo(self): + #self.setup_create_fact_dot() + + f = self.db['fact'].insert({ + 'hostname': TEST_FACT_DATA['hostname'], + 'fact': TEST_FACT_DATA['add_fact_data']['fact'], + 'timestamp': TEST_FACT_DATA['add_fact_data']['timestamp'], + 'host': TEST_FACT_DATA['add_fact_data']['host'].id, + 'module': TEST_FACT_DATA['add_fact_data']['module'] + }) + + q = { + '_id': f + } + # Bypass mongoengine and pymongo transform to get record + f_dict = self.db2['fact'].find_one(q) + self.assertIn('acpid3\uff0E4', f_dict['fact']) + + def test_fact_with_dot_deserialized_pymongo(self): + self.setup_create_fact_dot() + + q = { + '_id': self.f.id + } + f_dict = self.db['fact'].find_one(q) + self.assertIn('acpid3.4', f_dict['fact']) + + def test_fact_with_dot_deserialized(self): + self.setup_create_fact_dot() + + f = Fact.objects.get(id=self.f.id) + self.assertIn('acpid3.4', f.fact) + +class FactTransformUpdateTest(BaseFactTest): + pass diff --git a/awx/fact/tests/models/fact/fact_transform_pymongo.py b/awx/fact/tests/models/fact/fact_transform_pymongo.py new file mode 100644 index 0000000000..7cf81e4650 --- /dev/null +++ b/awx/fact/tests/models/fact/fact_transform_pymongo.py @@ -0,0 +1,96 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +from __future__ import absolute_import +from datetime import datetime + +# Django +from django.conf import settings + +# Pymongo +import pymongo + +# AWX +from awx.fact.models.fact import * # noqa +from .base import BaseFactTest + +__all__ = ['FactSerializePymongoTest', 'FactDeserializePymongoTest',] + +class FactPymongoBaseTest(BaseFactTest): + def setUp(self): + super(FactPymongoBaseTest, self).setUp() + # TODO: get host settings from config + self.client = pymongo.MongoClient('localhost', 27017) + self.db2 = self.client[settings.MONGO_DB] + + def _create_fact(self): + fact = {} + fact[self.k] = self.v + q = { + 'hostname': 'blah' + } + h = self.db['fact_host'].insert(q) + q = { + 'host': h, + 'module': 'blah', + 'timestamp': datetime.now(), + 'fact': fact + } + f = self.db['fact'].insert(q) + return f + + def check_transform(self, id): + raise RuntimeError("Must override") + + def create_dot_fact(self): + self.k = 'this.is.a.key' + self.v = 'this.is.a.value' + + self.k_uni = 'this\uff0Eis\uff0Ea\uff0Ekey' + + return self._create_fact() + + def create_dollar_fact(self): + self.k = 'this$is$a$key' + self.v = 'this$is$a$value' + + self.k_uni = 'this\uff04is\uff04a\uff04key' + + return self._create_fact() + +class FactSerializePymongoTest(FactPymongoBaseTest): + def check_transform(self, id): + q = { + '_id': id + } + f = self.db2.fact.find_one(q) + self.assertIn(self.k_uni, f['fact']) + self.assertEqual(f['fact'][self.k_uni], self.v) + + # Ensure key . are being transformed to the equivalent unicode into the database + def test_key_transform_dot(self): + f = self.create_dot_fact() + self.check_transform(f) + + # Ensure key $ are being transformed to the equivalent unicode into the database + def test_key_transform_dollar(self): + f = self.create_dollar_fact() + self.check_transform(f) + +class FactDeserializePymongoTest(FactPymongoBaseTest): + def check_transform(self, id): + q = { + '_id': id + } + f = self.db.fact.find_one(q) + self.assertIn(self.k, f['fact']) + self.assertEqual(f['fact'][self.k], self.v) + + def test_key_transform_dot(self): + f = self.create_dot_fact() + self.check_transform(f) + + def test_key_transform_dollar(self): + f = self.create_dollar_fact() + self.check_transform(f) diff --git a/awx/fact/tests/utils/dbtransform.py b/awx/fact/tests/utils/dbtransform.py index a11375223d..97bfad9c65 100644 --- a/awx/fact/tests/utils/dbtransform.py +++ b/awx/fact/tests/utils/dbtransform.py @@ -1,77 +1,112 @@ # Copyright (c) 2015 Ansible, Inc. # All Rights Reserved -# Python -from datetime import datetime -from mongoengine import connect - -# Django -from django.conf import settings - # AWX -from awx.main.tests.base import BaseTest, MongoDBRequired +from awx.main.tests.base import BaseTest from awx.fact.models.fact import * # noqa +from awx.fact.utils.dbtransform import KeyTransform -__all__ = ['DBTransformTest'] +#__all__ = ['DBTransformTest', 'KeyTransformUnitTest'] +__all__ = ['KeyTransformUnitTest'] -class DBTransformTest(BaseTest, MongoDBRequired): +class KeyTransformUnitTest(BaseTest): def setUp(self): - super(DBTransformTest, self).setUp() + super(KeyTransformUnitTest, self).setUp() + self.key_transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')]) - # Create a db connection that doesn't have the transformation registered - # Note: this goes through pymongo not mongoengine - self.client = connect(settings.MONGO_DB) - self.db = self.client[settings.MONGO_DB] + def test_no_replace(self): + value = { + "a_key_with_a_dict" : { + "key" : "value", + "nested_key_with_dict": { + "nested_key_with_value" : "deep_value" + } + } + } - def _create_fact(self): - fact = {} - fact[self.k] = self.v - h = FactHost(hostname='blah') - h.save() - f = Fact(host=h,module='blah',timestamp=datetime.now(),fact=fact) - f.save() - return f + data = self.key_transform.transform_incoming(value, None) + self.assertEqual(data, value) - def create_dot_fact(self): - self.k = 'this.is.a.key' - self.v = 'this.is.a.value' + data = self.key_transform.transform_outgoing(value, None) + self.assertEqual(data, value) - self.k_uni = 'this\uff0Eis\uff0Ea\uff0Ekey' + def test_complex(self): + value = { + "a.key.with.a.dict" : { + "key" : "value", + "nested.key.with.dict": { + "nested.key.with.value" : "deep_value" + } + } + } + value_transformed = { + "a\uff0Ekey\uff0Ewith\uff0Ea\uff0Edict" : { + "key" : "value", + "nested\uff0Ekey\uff0Ewith\uff0Edict": { + "nested\uff0Ekey\uff0Ewith\uff0Evalue" : "deep_value" + } + } + } - return self._create_fact() + data = self.key_transform.transform_incoming(value, None) + self.assertEqual(data, value_transformed) - def create_dollar_fact(self): - self.k = 'this$is$a$key' - self.v = 'this$is$a$value' + data = self.key_transform.transform_outgoing(value_transformed, None) + self.assertEqual(data, value) - self.k_uni = 'this\uff04is\uff04a\uff04key' + def test_simple(self): + value = { + "a.key" : "value" + } + value_transformed = { + "a\uff0Ekey" : "value" + } - return self._create_fact() + data = self.key_transform.transform_incoming(value, None) + self.assertEqual(data, value_transformed) - def check_unicode(self, f): - f_raw = self.db.fact.find_one(id=f.id) - self.assertIn(self.k_uni, f_raw['fact']) - self.assertEqual(f_raw['fact'][self.k_uni], self.v) + data = self.key_transform.transform_outgoing(value_transformed, None) + self.assertEqual(data, value) - # Ensure key . are being transformed to the equivalent unicode into the database - def test_key_transform_dot_unicode_in_storage(self): - f = self.create_dot_fact() - self.check_unicode(f) + def test_nested_dict(self): + value = { + "a.key.with.a.dict" : { + "nested.key." : "value" + } + } + value_transformed = { + "a\uff0Ekey\uff0Ewith\uff0Ea\uff0Edict" : { + "nested\uff0Ekey\uff0E" : "value" + } + } - # Ensure key $ are being transformed to the equivalent unicode into the database - def test_key_transform_dollar_unicode_in_storage(self): - f = self.create_dollar_fact() - self.check_unicode(f) + data = self.key_transform.transform_incoming(value, None) + self.assertEqual(data, value_transformed) + + data = self.key_transform.transform_outgoing(value_transformed, None) + self.assertEqual(data, value) - def check_transform(self): - f = Fact.objects.all()[0] - self.assertIn(self.k, f.fact) - self.assertEqual(f.fact[self.k], self.v) + def test_array(self): + value = { + "a.key.with.an.array" : [ + { + "key.with.dot" : "value" + } + ] + } + value_transformed = { + "a\uff0Ekey\uff0Ewith\uff0Ean\uff0Earray" : [ + { + "key\uff0Ewith\uff0Edot" : "value" + } + ] + } + data = self.key_transform.transform_incoming(value, None) + self.assertEqual(data, value_transformed) + + data = self.key_transform.transform_outgoing(value_transformed, None) + self.assertEqual(data, value) - def test_key_transform_dot_on_retreive(self): - self.create_dot_fact() - self.check_transform() - - def test_key_transform_dollar_on_retreive(self): - self.create_dollar_fact() - self.check_transform() +''' +class DBTransformTest(BaseTest, MongoDBRequired): +''' diff --git a/awx/fact/utils/dbtransform.py b/awx/fact/utils/dbtransform.py index f88708a467..98ce8180c9 100644 --- a/awx/fact/utils/dbtransform.py +++ b/awx/fact/utils/dbtransform.py @@ -1,55 +1,55 @@ # Copyright (c) 2014, Ansible, Inc. # All Rights Reserved. + +# Pymongo from pymongo.son_manipulator import SONManipulator -''' -Inspired by: https://stackoverflow.com/questions/8429318/how-to-use-dot-in-field-name/20698802#20698802 - -Replace . and $ with unicode values -''' class KeyTransform(SONManipulator): + def __init__(self, replace): self.replace = replace - def transform_key(self, key, replace, replacement): - """Transform key for saving to database.""" - return key.replace(replace, replacement) + def replace_key(self, key): + for (replace, replacement) in self.replace: + key = key.replace(replace, replacement) + return key - def revert_key(self, key, replace, replacement): - """Restore transformed key returning from database.""" - return key.replace(replacement, replace) + def revert_key(self, key): + for (replacement, replace) in self.replace: + key = key.replace(replace, replacement) + return key + + def replace_incoming(self, obj): + if isinstance(obj, dict): + value = {} + for k, v in obj.items(): + value[self.replace_key(k)] = self.replace_incoming(v) + elif isinstance(obj, list): + value = [self.replace_incoming(elem) + for elem in obj] + else: + value = obj + + return value + + def replace_outgoing(self, obj): + if isinstance(obj, dict): + value = {} + for k, v in obj.items(): + value[self.revert_key(k)] = self.replace_outgoing(v) + elif isinstance(obj, list): + value = [self.replace_outgoing(elem) + for elem in obj] + else: + value = obj + + return value def transform_incoming(self, son, collection): - """Recursively replace all keys that need transforming.""" - for (key, value) in son.items(): - for r in self.replace: - replace = r[0] - replacement = r[1] - if replace in key: - if isinstance(value, dict): - son[self.transform_key(key, replace, replacement)] = self.transform_incoming( - son.pop(key), collection) - else: - son[self.transform_key(key, replace, replacement)] = son.pop(key) - elif isinstance(value, dict): # recurse into sub-docs - son[key] = self.transform_incoming(value, collection) - return son + return self.replace_incoming(son) def transform_outgoing(self, son, collection): - """Recursively restore all transformed keys.""" - for (key, value) in son.items(): - for r in self.replace: - replace = r[0] - replacement = r[1] - if replacement in key: - if isinstance(value, dict): - son[self.revert_key(key, replace, replacement)] = self.transform_outgoing( - son.pop(key), collection) - else: - son[self.revert_key(key, replace, replacement)] = son.pop(key) - elif isinstance(value, dict): # recurse into sub-docs - son[key] = self.transform_outgoing(value, collection) - return son + return self.replace_outgoing(son) def register_key_transform(db): db.add_son_manipulator(KeyTransform([('.', '\uff0E'), ('$', '\uff04')])) diff --git a/awx/main/tests/commands/run_fact_cache_receiver.py b/awx/main/tests/commands/run_fact_cache_receiver.py index 1a63b25180..3933813ff4 100644 --- a/awx/main/tests/commands/run_fact_cache_receiver.py +++ b/awx/main/tests/commands/run_fact_cache_receiver.py @@ -182,7 +182,7 @@ class RunFactCacheReceiverUnitTest(BaseTest, MongoDBRequired): def test_process_facts_message_ansible_overwrite(self): data = copy_only_module(TEST_MSG, 'ansible') - key = 'ansible_overwrite' + key = 'ansible.overwrite' value = 'hello world' receiver = FactCacheReceiver() @@ -197,3 +197,4 @@ class RunFactCacheReceiverUnitTest(BaseTest, MongoDBRequired): fact = Fact.objects.get(id=fact.id) self.assertIn(key, fact.fact) self.assertEqual(fact.fact[key], value) + self.assertEqual(fact.fact, data['facts'])