Merge pull request #130 from chrismeyersfsu/feature-mongodb_single_fact

added single fact across multiple hosts query + test cases

awx/fact/models/fact.py

@@ -1,3 +1,6 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
 from mongoengine import Document, DynamicDocument, DateTimeField, ReferenceField, StringField
 
 class FactHost(Document):
@@ -79,6 +82,35 @@ class Fact(DynamicDocument):
 
         return FactVersion.objects.filter(**kv).values_list('timestamp')
 
+    @staticmethod
+    def get_single_facts(hostnames, fact_key, timestamp, module):
+        host_ids = FactHost.objects.filter(hostname__in=hostnames).values_list('id')
+        if not host_ids or len(host_ids) == 0:
+            return None
+
+        kv = {
+            'host__in': host_ids,
+            'timestamp__lte': timestamp,
+            'module': module,
+        }
+        facts = FactVersion.objects.filter(**kv).values_list('fact')
+        if not facts or len(facts) == 0:
+            return None
+        # TODO: Make sure the below doesn't trigger a query to get the fact record
+        # It's unclear as to if mongoengine will query the full fact when the id is referenced.
+        # This is not a logic problem, but a performance problem.
+        fact_ids = [fact.id for fact in facts]
+
+        project = {
+            '$project': {
+                'host': 1,
+                'fact.%s' % fact_key: 1,
+            }
+        }
+        facts = Fact.objects.filter(id__in=fact_ids).aggregate(project)
+        return facts
+
+
 class FactVersion(Document):
     timestamp = DateTimeField(required=True)
     host = ReferenceField(FactHost, required=True)

awx/fact/tests/models/fact/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+from __future__ import absolute_import
+
+from .fact_simple import * # noqa
+from .fact_get_single_facts import * # noqa

awx/fact/tests/models/fact/base.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+# Python
+from __future__ import absolute_import
+from awx.main.tests.base import BaseTest, MongoDBRequired
+
+# AWX
+from awx.fact.models.fact import * # noqa
+
+'''
+Helper functions (i.e. create_host_document) expect the structure:
+{
+    'hostname': 'hostname1',
+    'add_fact_data': {
+        'timestamp': datetime.now(),
+        'host': None,
+        'module': 'packages',
+        'fact': ...
+    }
+}
+'''
+class BaseFactTest(BaseTest, MongoDBRequired):
+
+    @staticmethod
+    def _normalize_timestamp(timestamp):
+        return timestamp.replace(microsecond=0)
+
+    @staticmethod
+    def normalize_timestamp(data):
+        data['add_fact_data']['timestamp'] = BaseFactTest._normalize_timestamp(data['add_fact_data']['timestamp'])
+
+    def create_host_document(self, data):
+        data['add_fact_data']['host'] = FactHost(hostname=data['hostname']).save()

awx/fact/tests/models/fact/fact_get_single_facts.py

@@ -0,0 +1,106 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+# Python
+from __future__ import absolute_import
+from datetime import datetime
+from copy import deepcopy
+
+# Django
+
+# AWX
+from awx.fact.models.fact import * # noqa
+from .base import BaseFactTest
+
+__all__ = ['FactGetSingleFactsTest']
+
+TEST_FACT_DATA = {
+    'hostname': 'hostname_%d',
+    'add_fact_data': {
+        'timestamp': datetime.now(),
+        'host': None,
+        'module': 'packages',
+        'fact': {
+            "accountsservice": [
+                {
+                    "architecture": "amd64",
+                    "name": "accountsservice",
+                    "source": "apt",
+                    "version": "0.6.35-0ubuntu7.1"
+                }
+            ],
+            "acpid": [
+                {
+                    "architecture": "amd64",
+                    "name": "acpid",
+                    "source": "apt",
+                    "version": "1:2.0.21-1ubuntu2"
+                }
+            ],
+            "adduser": [
+                {
+                    "architecture": "all",
+                    "name": "adduser",
+                    "source": "apt",
+                    "version": "3.113+nmu3ubuntu3"
+                }
+            ],
+        },
+    }
+}
+
+
+class FactGetSingleFactsTest(BaseFactTest):
+    def create_fact_scans_unique_hosts(self, host_count):
+        self.fact_data = []
+        self.fact_objs = []
+        self.hostnames = []
+        for i in range(1, host_count+1):
+            fact_data = deepcopy(TEST_FACT_DATA)
+            fact_data['hostname'] = fact_data['hostname'] % (i)
+            fact_data['add_fact_data']['timestamp'] = datetime.now().replace(year=2015 - i)
+            BaseFactTest.normalize_timestamp(fact_data)
+
+            self.create_host_document(fact_data)
+            (fact_obj, version_obj) = Fact.add_fact(**fact_data['add_fact_data'])
+
+            self.fact_data.append(fact_data)
+            self.fact_objs.append(fact_obj)
+            self.hostnames.append(fact_data['hostname'])
+
+    def setUp(self):
+        super(FactGetSingleFactsTest, self).setUp()
+        self.host_count = 20
+        self.create_fact_scans_unique_hosts(self.host_count)
+
+    def check_query_results(self, facts_known, facts):
+        # Transpose facts to a dict with key _id
+        count = 0
+        facts_dict = {}
+        for fact in facts:
+            count += 1
+            facts_dict[fact['_id']] = fact
+        self.assertEqual(count, len(facts_known))
+
+        # For each fact that we put into the database on setup,
+        # we should find that fact in the result set returned
+        for fact_known in facts_known:
+            key = fact_known.id
+            self.assertIn(key, facts_dict)
+            self.assertEqual(facts_dict[key]['fact']['acpid'], fact_known.fact['acpid'])
+            self.assertEqual(facts_dict[key]['host'], fact_known.host.id)
+
+    def test_get_single_facts_ok(self):
+        timestamp = datetime.now().replace(year=2016)
+        facts = Fact.get_single_facts(self.hostnames, 'acpid', timestamp, 'packages')
+        self.assertIsNotNone(facts)
+
+        self.check_query_results(self.fact_objs, facts)
+
+    def test_get_single_facts_subset_by_timestamp(self):
+        timestamp = datetime.now().replace(year=2010)
+        facts = Fact.get_single_facts(self.hostnames, 'acpid', timestamp, 'packages')
+        self.assertIsNotNone(facts)
+
+        self.check_query_results(self.fact_objs[4:], facts)
+        

awx/fact/tests/models/fact/fact_simple.py

@@ -2,6 +2,7 @@
 # All Rights Reserved
 
 # Python
+from __future__ import absolute_import
 from datetime import datetime
 from copy import deepcopy
 
@@ -10,8 +11,9 @@ from copy import deepcopy
 # AWX
 from awx.fact.models.fact import * # noqa
 from awx.main.tests.base import BaseTest, MongoDBRequired
+from .base import BaseFactTest
 
-__all__ = ['FactHostTest', 'FactTest', 'FactGetHostVersionTest', 'FactGetHostTimeline']
+__all__ = ['FactHostTest', 'FactTest', 'FactGetHostVersionTest', 'FactGetHostTimelineTest']
 
 TEST_FACT_DATA = {
     'hostname': 'hostname1',
@@ -48,10 +50,7 @@ TEST_FACT_DATA = {
     }
 }
 # Strip off microseconds because mongo has less precision
-TEST_FACT_DATA['add_fact_data']['timestamp'] = TEST_FACT_DATA['add_fact_data']['timestamp'].replace(microsecond=0)
+BaseFactTest.normalize_timestamp(TEST_FACT_DATA)
-
-def create_host_document():
-    TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save()
 
 def create_fact_scans(count=1):
     timestamps = []
@@ -65,7 +64,7 @@ def create_fact_scans(count=1):
     return timestamps
 
 
-class FactHostTest(BaseTest, MongoDBRequired):
+class FactHostTest(BaseFactTest):
     def test_create_host(self):
         host = FactHost(hostname=TEST_FACT_DATA['hostname'])
         host.save()
@@ -81,10 +80,10 @@ class FactHostTest(BaseTest, MongoDBRequired):
 
         self.assertRaises(FactHost.DoesNotExist, FactHost.objects.get, hostname='doesnotexist')
 
-class FactTest(BaseTest, MongoDBRequired):
+class FactTest(BaseFactTest):
     def setUp(self):
         super(FactTest, self).setUp()
-        create_host_document()
+        self.create_host_document(TEST_FACT_DATA)
 
     def test_add_fact(self):
         (f_obj, v_obj) = Fact.add_fact(**TEST_FACT_DATA['add_fact_data'])
@@ -106,10 +105,10 @@ class FactTest(BaseTest, MongoDBRequired):
         self.assertEqual(v.fact.id, f_obj.id)
         self.assertEqual(v.fact.module, TEST_FACT_DATA['add_fact_data']['module'])
 
-class FactGetHostVersionTest(BaseTest, MongoDBRequired):
+class FactGetHostVersionTest(BaseFactTest):
     def setUp(self):
         super(FactGetHostVersionTest, self).setUp()
-        create_host_document()
+        self.create_host_document(TEST_FACT_DATA)
 
         self.t1 = datetime.now().replace(second=1, microsecond=0)
         self.t2 = datetime.now().replace(second=2, microsecond=0)
@@ -137,10 +136,10 @@ class FactGetHostVersionTest(BaseTest, MongoDBRequired):
         fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module'])
         self.assertIsNone(fact)
 
-class FactGetHostTimeline(BaseTest, MongoDBRequired):
+class FactGetHostTimelineTest(BaseFactTest):
     def setUp(self):
-        super(FactGetHostTimeline, self).setUp()
+        super(FactGetHostTimelineTest, self).setUp()
-        create_host_document()
+        self.create_host_document(TEST_FACT_DATA)
 
         self.scans = 20
         self.timestamps = create_fact_scans(self.scans)
@@ -151,4 +150,3 @@ class FactGetHostTimeline(BaseTest, MongoDBRequired):
         self.assertEqual(len(timestamps), len(self.timestamps))
         for i in range(0, self.scans):
             self.assertEqual(timestamps[i], self.timestamps[i])
-