moved new fact implementation to fact app

awx/fact/__init__.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2014 AnsibleWorks, Inc.
+# All Rights Reserved.
+
+from __future__ import absolute_import
+
+import logging
+from django.conf import settings
+
+from mongoengine import connect
+from mongoengine.connection import get_db, ConnectionError
+from .utils.dbtransform import register_key_transform
+
+logger = logging.getLogger('fact.__init__')
+
+# Connect to Mongo
+try:
+    connect(settings.MONGO_DB)
+    register_key_transform(get_db())
+except ConnectionError:
+    logger.warn('Failed to establish connect to MongDB "%s"' % (settings.MONGO_DB))

awx/fact/models/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+from __future__ import absolute_import
+
+from .fact import * # noqa

awx/fact/models/fact.py

@@ -0,0 +1,95 @@
+from mongoengine import Document, DynamicDocument, DateTimeField, ReferenceField, StringField
+
+class FactHost(Document):
+    hostname = StringField(max_length=100, required=True, unique=True)
+
+    # TODO: Consider using hashed index on hostname. django-mongo may not support this but
+    # executing raw js will
+    meta = {
+        'indexes': [
+            'hostname' 
+        ]
+    }
+
+    @staticmethod
+    def get_host_id(hostname):
+        host = FactHost.objects.get(hostname=hostname)
+        if host:
+            return host.id
+        return None
+
+class Fact(DynamicDocument):
+    timestamp = DateTimeField(required=True)
+    host = ReferenceField(FactHost, required=True)
+    module = StringField(max_length=50, required=True)
+    # fact = <anything>
+
+    # TODO: Consider using hashed index on host. django-mongo may not support this but
+    # executing raw js will
+    meta = {
+        'indexes': [
+            '-timestamp',
+            'host'
+        ]
+    }
+
+    @staticmethod
+    def add_fact(timestamp, fact, host, module):
+        fact_obj = Fact(timestamp=timestamp, host=host, module=module, fact=fact)
+        fact_obj.save()
+        version_obj = FactVersion(timestamp=timestamp, host=host, module=module, fact=fact_obj)
+        version_obj.save()
+        return (fact_obj, version_obj)
+
+    # TODO: if we want to relax the need to include module...
+    # If module not specified then filter query may return more than 1 result.
+    # Thus, the resulting facts must somehow be unioned/concated/ or kept as an array.
+    @staticmethod
+    def get_host_version(hostname, timestamp, module):
+        try:
+            host = FactHost.objects.get(hostname=hostname)
+        except FactHost.DoesNotExist:
+            return None
+
+        kv = {
+            'host' : host.id,
+            'timestamp__lte': timestamp,
+            'module': module,
+        }
+
+        try:
+            facts = Fact.objects.filter(**kv)
+            if not facts:
+                return None
+            return facts[0]
+        except Fact.DoesNotExist:
+            return None
+
+    @staticmethod
+    def get_host_timeline(hostname, module):
+        try:
+            host = FactHost.objects.get(hostname=hostname)
+        except FactHost.DoesNotExist:
+            return None
+
+        kv = {
+            'host': host.id,
+            'module': module,
+        }
+
+        return FactVersion.objects.filter(**kv).values_list('timestamp')
+
+class FactVersion(Document):
+    timestamp = DateTimeField(required=True)
+    host = ReferenceField(FactHost, required=True)
+    module = StringField(max_length=50, required=True)  
+    fact = ReferenceField(Fact, required=True)
+    # TODO: Consider using hashed index on module. django-mongo may not support this but
+    # executing raw js will
+    meta = {
+        'indexes': [
+            '-timestamp',
+            'module'
+        ]
+    }
+    

awx/fact/tests/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+from __future__ import absolute_import
+
+from .models import * # noqa
+from .utils import * # noqa

awx/fact/tests/models/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+from __future__ import absolute_import
+
+from .fact import * # noqa

awx/fact/tests/models/fact.py

@@ -0,0 +1,154 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+# Python
+from datetime import datetime
+from copy import deepcopy
+
+# Django
+
+# AWX
+from awx.fact.models.fact import * # noqa
+from awx.main.tests.base import BaseTest, MongoDBRequired
+
+__all__ = ['FactHostTest', 'FactTest', 'FactGetHostVersionTest', 'FactGetHostTimeline']
+
+TEST_FACT_DATA = {
+    'hostname': 'hostname1',
+    'add_fact_data': {
+        'timestamp': datetime.now(),
+        'host': None,
+        'module': 'packages',
+        'fact': {
+            "accountsservice": [
+                {
+                    "architecture": "amd64",
+                    "name": "accountsservice",
+                    "source": "apt",
+                    "version": "0.6.35-0ubuntu7.1"
+                }
+            ],
+            "acpid": [
+                {
+                    "architecture": "amd64",
+                    "name": "acpid",
+                    "source": "apt",
+                    "version": "1:2.0.21-1ubuntu2"
+                }
+            ],
+            "adduser": [
+                {
+                    "architecture": "all",
+                    "name": "adduser",
+                    "source": "apt",
+                    "version": "3.113+nmu3ubuntu3"
+                }
+            ],
+        },
+    }
+}
+# Strip off microseconds because mongo has less precision
+TEST_FACT_DATA['add_fact_data']['timestamp'] = TEST_FACT_DATA['add_fact_data']['timestamp'].replace(microsecond=0)
+
+def create_host_document():
+    TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save()
+
+def create_fact_scans(count=1):
+    timestamps = []
+    for i in range(0, count):
+        data = deepcopy(TEST_FACT_DATA)
+        t = datetime.now().replace(year=2015 - i, microsecond=0)
+        data['add_fact_data']['timestamp'] = t
+        (f, v) = Fact.add_fact(**data['add_fact_data'])
+        timestamps.append(t)
+
+    return timestamps
+
+
+class FactHostTest(BaseTest, MongoDBRequired):
+    def test_create_host(self):
+        host = FactHost(hostname=TEST_FACT_DATA['hostname'])
+        host.save()
+
+        host = FactHost.objects.get(hostname=TEST_FACT_DATA['hostname'])
+        self.assertIsNotNone(host, "Host added but not found")
+        self.assertEqual(TEST_FACT_DATA['hostname'], host.hostname, "Gotten record hostname does not match expected hostname")
+
+    # Ensure an error is raised for .get() that doesn't match a record.
+    def test_get_host_id_no_result(self):
+        host = FactHost(hostname=TEST_FACT_DATA['hostname'])
+        host.save()
+
+        self.assertRaises(FactHost.DoesNotExist, FactHost.objects.get, hostname='doesnotexist')
+
+class FactTest(BaseTest, MongoDBRequired):
+    def setUp(self):
+        super(FactTest, self).setUp()
+        create_host_document()
+
+    def test_add_fact(self):
+        (f_obj, v_obj) = Fact.add_fact(**TEST_FACT_DATA['add_fact_data'])
+        f = Fact.objects.get(id=f_obj.id)
+        v = FactVersion.objects.get(id=v_obj.id)
+
+        self.assertEqual(f.id, f_obj.id)
+        self.assertEqual(f.module, TEST_FACT_DATA['add_fact_data']['module'])
+        self.assertEqual(f.fact, TEST_FACT_DATA['add_fact_data']['fact'])
+        self.assertEqual(f.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp'])
+
+        # host relationship created
+        self.assertEqual(f.host.id, TEST_FACT_DATA['add_fact_data']['host'].id)
+
+        # version created and related
+        self.assertEqual(v.id, v_obj.id)
+        self.assertEqual(v.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp'])
+        self.assertEqual(v.host.id, TEST_FACT_DATA['add_fact_data']['host'].id)
+        self.assertEqual(v.fact.id, f_obj.id)
+        self.assertEqual(v.fact.module, TEST_FACT_DATA['add_fact_data']['module'])
+
+class FactGetHostVersionTest(BaseTest, MongoDBRequired):
+    def setUp(self):
+        super(FactGetHostVersionTest, self).setUp()
+        create_host_document()
+
+        self.t1 = datetime.now().replace(second=1, microsecond=0)
+        self.t2 = datetime.now().replace(second=2, microsecond=0)
+        data = deepcopy(TEST_FACT_DATA)
+        data['add_fact_data']['timestamp'] = self.t1
+        (self.f1, self.v1) = Fact.add_fact(**data['add_fact_data'])
+        data = deepcopy(TEST_FACT_DATA)
+        data['add_fact_data']['timestamp'] = self.t2
+        (self.f2, self.v2) = Fact.add_fact(**data['add_fact_data'])
+
+    def test_get_host_version_exact_timestamp(self):
+        fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=self.t1, module=TEST_FACT_DATA['add_fact_data']['module'])
+        self.assertIsNotNone(fact, "Set of Facts not found")
+        self.assertEqual(self.f1.id, fact.id)
+        self.assertEqual(self.f1.fact, fact.fact)
+
+    def test_get_host_version_lte_timestamp(self):
+        t3 = datetime.now().replace(second=3, microsecond=0)
+        fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module'])
+        self.assertEqual(self.f1.id, fact.id)
+        self.assertEqual(self.f1.fact, fact.fact)
+
+    def test_get_host_version_none(self):
+        t3 = deepcopy(self.t1).replace(second=0)
+        fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module'])
+        self.assertIsNone(fact)
+
+class FactGetHostTimeline(BaseTest, MongoDBRequired):
+    def setUp(self):
+        super(FactGetHostTimeline, self).setUp()
+        create_host_document()
+
+        self.scans = 20
+        self.timestamps = create_fact_scans(self.scans)
+
+    def test_get_host_timeline_ok(self):
+        timestamps = Fact.get_host_timeline(hostname=TEST_FACT_DATA['hostname'], module=TEST_FACT_DATA['add_fact_data']['module'])
+        self.assertIsNotNone(timestamps)
+        self.assertEqual(len(timestamps), len(self.timestamps))
+        for i in range(0, self.scans):
+            self.assertEqual(timestamps[i], self.timestamps[i])
+

awx/fact/tests/utils/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+from __future__ import absolute_import
+
+from .dbtransform import * # noqa

awx/fact/tests/utils/dbtransform.py

@@ -0,0 +1,77 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+# Python
+from datetime import datetime
+from mongoengine import connect
+
+# Django
+from django.conf import settings
+
+# AWX
+from awx.main.tests.base import BaseTest, MongoDBRequired
+from awx.fact.models.fact import * # noqa
+
+__all__ = ['DBTransformTest']
+
+class DBTransformTest(BaseTest, MongoDBRequired):
+    def setUp(self):
+        super(DBTransformTest, self).setUp()
+
+        # Create a db connection that doesn't have the transformation registered
+        # Note: this goes through pymongo not mongoengine
+        self.client = connect(settings.MONGO_DB)
+        self.db = self.client[settings.MONGO_DB]
+
+    def _create_fact(self):
+        fact = {}
+        fact[self.k] = self.v
+        h = FactHost(hostname='blah')
+        h.save()
+        f = Fact(host=h,module='blah',timestamp=datetime.now(),fact=fact)
+        f.save()
+        return f
+
+    def create_dot_fact(self):
+        self.k = 'this.is.a.key'
+        self.v = 'this.is.a.value'
+
+        self.k_uni = 'this\uff0Eis\uff0Ea\uff0Ekey'
+
+        return self._create_fact()
+
+    def create_dollar_fact(self):
+        self.k = 'this$is$a$key'
+        self.v = 'this$is$a$value'
+
+        self.k_uni = 'this\uff04is\uff04a\uff04key'
+
+        return self._create_fact()
+
+    def check_unicode(self, f):
+        f_raw = self.db.fact.find_one(id=f.id)
+        self.assertIn(self.k_uni, f_raw['fact'])
+        self.assertEqual(f_raw['fact'][self.k_uni], self.v)
+
+    # Ensure key . are being transformed to the equivalent unicode into the database
+    def test_key_transform_dot_unicode_in_storage(self):
+        f = self.create_dot_fact()
+        self.check_unicode(f)
+
+    # Ensure key $ are being transformed to the equivalent unicode into the database
+    def test_key_transform_dollar_unicode_in_storage(self):
+        f = self.create_dollar_fact()
+        self.check_unicode(f)
+
+    def check_transform(self):
+        f = Fact.objects.all()[0]
+        self.assertIn(self.k, f.fact)
+        self.assertEqual(f.fact[self.k], self.v)
+
+    def test_key_transform_dot_on_retreive(self):
+        self.create_dot_fact()
+        self.check_transform()
+
+    def test_key_transform_dollar_on_retreive(self):
+        self.create_dollar_fact()
+        self.check_transform()

awx/fact/utils/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved

awx/fact/utils/dbtransform.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2014, Ansible, Inc.
+# All Rights Reserved.
+from pymongo.son_manipulator import SONManipulator
+
+'''
+Inspired by: https://stackoverflow.com/questions/8429318/how-to-use-dot-in-field-name/20698802#20698802
+
+Replace . and $ with unicode values
+'''
+class KeyTransform(SONManipulator):
+    def __init__(self, replace):
+        self.replace = replace
+
+    def transform_key(self, key, replace, replacement):
+        """Transform key for saving to database."""
+        return key.replace(replace, replacement)
+
+    def revert_key(self, key, replace, replacement):
+        """Restore transformed key returning from database."""
+        return key.replace(replacement, replace)
+
+    def transform_incoming(self, son, collection):
+        """Recursively replace all keys that need transforming."""
+        for (key, value) in son.items():
+            for r in self.replace:
+                replace = r[0]
+                replacement = r[1]
+                if replace in key:
+                    if isinstance(value, dict):
+                        son[self.transform_key(key, replace, replacement)] = self.transform_incoming(
+                            son.pop(key), collection)
+                    else:
+                        son[self.transform_key(key, replace, replacement)] = son.pop(key)
+                elif isinstance(value, dict):  # recurse into sub-docs
+                    son[key] = self.transform_incoming(value, collection)
+        return son
+
+    def transform_outgoing(self, son, collection):
+        """Recursively restore all transformed keys."""
+        for (key, value) in son.items():
+            for r in self.replace:
+                replace = r[0]
+                replacement = r[1]
+                if replacement in key:
+                    if isinstance(value, dict):
+                        son[self.revert_key(key, replace, replacement)] = self.transform_outgoing(
+                            son.pop(key), collection)
+                    else:
+                        son[self.revert_key(key, replace, replacement)] = son.pop(key)
+                elif isinstance(value, dict):  # recurse into sub-docs
+                    son[key] = self.transform_outgoing(value, collection)
+        return son
+
+def register_key_transform(db):
+    db.add_son_manipulator(KeyTransform([('.', '\uff0E'), ('$', '\uff04')]))