From fc73f1f87aa859db8ee510f22477eecd1737f6be Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Tue, 7 Apr 2015 17:50:07 -0400 Subject: [PATCH] added single fact across multiple hosts query + test cases --- awx/fact/models/fact.py | 32 ++++++ awx/fact/tests/models/fact/__init__.py | 7 ++ awx/fact/tests/models/fact/base.py | 34 ++++++ .../models/fact/fact_get_single_facts.py | 106 ++++++++++++++++++ .../models/{fact.py => fact/fact_simple.py} | 26 ++--- 5 files changed, 191 insertions(+), 14 deletions(-) create mode 100644 awx/fact/tests/models/fact/__init__.py create mode 100644 awx/fact/tests/models/fact/base.py create mode 100644 awx/fact/tests/models/fact/fact_get_single_facts.py rename awx/fact/tests/models/{fact.py => fact/fact_simple.py} (89%) diff --git a/awx/fact/models/fact.py b/awx/fact/models/fact.py index 0887fca171..63bb08dfcc 100644 --- a/awx/fact/models/fact.py +++ b/awx/fact/models/fact.py @@ -1,3 +1,6 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + from mongoengine import Document, DynamicDocument, DateTimeField, ReferenceField, StringField class FactHost(Document): @@ -79,6 +82,35 @@ class Fact(DynamicDocument): return FactVersion.objects.filter(**kv).values_list('timestamp') + @staticmethod + def get_single_facts(hostnames, fact_key, timestamp, module): + host_ids = FactHost.objects.filter(hostname__in=hostnames).values_list('id') + if not host_ids or len(host_ids) == 0: + return None + + kv = { + 'host__in': host_ids, + 'timestamp__lte': timestamp, + 'module': module, + } + facts = FactVersion.objects.filter(**kv).values_list('fact') + if not facts or len(facts) == 0: + return None + # TODO: Make sure the below doesn't trigger a query to get the fact record + # It's unclear as to if mongoengine will query the full fact when the id is referenced. + # This is not a logic problem, but a performance problem. + fact_ids = [fact.id for fact in facts] + + project = { + '$project': { + 'host': 1, + 'fact.%s' % fact_key: 1, + } + } + facts = Fact.objects.filter(id__in=fact_ids).aggregate(project) + return facts + + class FactVersion(Document): timestamp = DateTimeField(required=True) host = ReferenceField(FactHost, required=True) diff --git a/awx/fact/tests/models/fact/__init__.py b/awx/fact/tests/models/fact/__init__.py new file mode 100644 index 0000000000..151002610f --- /dev/null +++ b/awx/fact/tests/models/fact/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +from __future__ import absolute_import + +from .fact_simple import * # noqa +from .fact_get_single_facts import * # noqa diff --git a/awx/fact/tests/models/fact/base.py b/awx/fact/tests/models/fact/base.py new file mode 100644 index 0000000000..3d8c4653f0 --- /dev/null +++ b/awx/fact/tests/models/fact/base.py @@ -0,0 +1,34 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +from __future__ import absolute_import +from awx.main.tests.base import BaseTest, MongoDBRequired + +# AWX +from awx.fact.models.fact import * # noqa + +''' +Helper functions (i.e. create_host_document) expect the structure: +{ + 'hostname': 'hostname1', + 'add_fact_data': { + 'timestamp': datetime.now(), + 'host': None, + 'module': 'packages', + 'fact': ... + } +} +''' +class BaseFactTest(BaseTest, MongoDBRequired): + + @staticmethod + def _normalize_timestamp(timestamp): + return timestamp.replace(microsecond=0) + + @staticmethod + def normalize_timestamp(data): + data['add_fact_data']['timestamp'] = BaseFactTest._normalize_timestamp(data['add_fact_data']['timestamp']) + + def create_host_document(self, data): + data['add_fact_data']['host'] = FactHost(hostname=data['hostname']).save() diff --git a/awx/fact/tests/models/fact/fact_get_single_facts.py b/awx/fact/tests/models/fact/fact_get_single_facts.py new file mode 100644 index 0000000000..f305dc7c51 --- /dev/null +++ b/awx/fact/tests/models/fact/fact_get_single_facts.py @@ -0,0 +1,106 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +from __future__ import absolute_import +from datetime import datetime +from copy import deepcopy + +# Django + +# AWX +from awx.fact.models.fact import * # noqa +from .base import BaseFactTest + +__all__ = ['FactGetSingleFactsTest'] + +TEST_FACT_DATA = { + 'hostname': 'hostname_%d', + 'add_fact_data': { + 'timestamp': datetime.now(), + 'host': None, + 'module': 'packages', + 'fact': { + "accountsservice": [ + { + "architecture": "amd64", + "name": "accountsservice", + "source": "apt", + "version": "0.6.35-0ubuntu7.1" + } + ], + "acpid": [ + { + "architecture": "amd64", + "name": "acpid", + "source": "apt", + "version": "1:2.0.21-1ubuntu2" + } + ], + "adduser": [ + { + "architecture": "all", + "name": "adduser", + "source": "apt", + "version": "3.113+nmu3ubuntu3" + } + ], + }, + } +} + + +class FactGetSingleFactsTest(BaseFactTest): + def create_fact_scans_unique_hosts(self, host_count): + self.fact_data = [] + self.fact_objs = [] + self.hostnames = [] + for i in range(1, host_count+1): + fact_data = deepcopy(TEST_FACT_DATA) + fact_data['hostname'] = fact_data['hostname'] % (i) + fact_data['add_fact_data']['timestamp'] = datetime.now().replace(year=2015 - i) + BaseFactTest.normalize_timestamp(fact_data) + + self.create_host_document(fact_data) + (fact_obj, version_obj) = Fact.add_fact(**fact_data['add_fact_data']) + + self.fact_data.append(fact_data) + self.fact_objs.append(fact_obj) + self.hostnames.append(fact_data['hostname']) + + def setUp(self): + super(FactGetSingleFactsTest, self).setUp() + self.host_count = 20 + self.create_fact_scans_unique_hosts(self.host_count) + + def check_query_results(self, facts_known, facts): + # Transpose facts to a dict with key _id + count = 0 + facts_dict = {} + for fact in facts: + count += 1 + facts_dict[fact['_id']] = fact + self.assertEqual(count, len(facts_known)) + + # For each fact that we put into the database on setup, + # we should find that fact in the result set returned + for fact_known in facts_known: + key = fact_known.id + self.assertIn(key, facts_dict) + self.assertEqual(facts_dict[key]['fact']['acpid'], fact_known.fact['acpid']) + self.assertEqual(facts_dict[key]['host'], fact_known.host.id) + + def test_get_single_facts_ok(self): + timestamp = datetime.now().replace(year=2016) + facts = Fact.get_single_facts(self.hostnames, 'acpid', timestamp, 'packages') + self.assertIsNotNone(facts) + + self.check_query_results(self.fact_objs, facts) + + def test_get_single_facts_subset_by_timestamp(self): + timestamp = datetime.now().replace(year=2010) + facts = Fact.get_single_facts(self.hostnames, 'acpid', timestamp, 'packages') + self.assertIsNotNone(facts) + + self.check_query_results(self.fact_objs[4:], facts) + \ No newline at end of file diff --git a/awx/fact/tests/models/fact.py b/awx/fact/tests/models/fact/fact_simple.py similarity index 89% rename from awx/fact/tests/models/fact.py rename to awx/fact/tests/models/fact/fact_simple.py index ffcd38857d..e61a613cca 100644 --- a/awx/fact/tests/models/fact.py +++ b/awx/fact/tests/models/fact/fact_simple.py @@ -2,6 +2,7 @@ # All Rights Reserved # Python +from __future__ import absolute_import from datetime import datetime from copy import deepcopy @@ -10,8 +11,9 @@ from copy import deepcopy # AWX from awx.fact.models.fact import * # noqa from awx.main.tests.base import BaseTest, MongoDBRequired +from .base import BaseFactTest -__all__ = ['FactHostTest', 'FactTest', 'FactGetHostVersionTest', 'FactGetHostTimeline'] +__all__ = ['FactHostTest', 'FactTest', 'FactGetHostVersionTest', 'FactGetHostTimelineTest'] TEST_FACT_DATA = { 'hostname': 'hostname1', @@ -48,10 +50,7 @@ TEST_FACT_DATA = { } } # Strip off microseconds because mongo has less precision -TEST_FACT_DATA['add_fact_data']['timestamp'] = TEST_FACT_DATA['add_fact_data']['timestamp'].replace(microsecond=0) - -def create_host_document(): - TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save() +BaseFactTest.normalize_timestamp(TEST_FACT_DATA) def create_fact_scans(count=1): timestamps = [] @@ -65,7 +64,7 @@ def create_fact_scans(count=1): return timestamps -class FactHostTest(BaseTest, MongoDBRequired): +class FactHostTest(BaseFactTest): def test_create_host(self): host = FactHost(hostname=TEST_FACT_DATA['hostname']) host.save() @@ -81,10 +80,10 @@ class FactHostTest(BaseTest, MongoDBRequired): self.assertRaises(FactHost.DoesNotExist, FactHost.objects.get, hostname='doesnotexist') -class FactTest(BaseTest, MongoDBRequired): +class FactTest(BaseFactTest): def setUp(self): super(FactTest, self).setUp() - create_host_document() + self.create_host_document(TEST_FACT_DATA) def test_add_fact(self): (f_obj, v_obj) = Fact.add_fact(**TEST_FACT_DATA['add_fact_data']) @@ -106,10 +105,10 @@ class FactTest(BaseTest, MongoDBRequired): self.assertEqual(v.fact.id, f_obj.id) self.assertEqual(v.fact.module, TEST_FACT_DATA['add_fact_data']['module']) -class FactGetHostVersionTest(BaseTest, MongoDBRequired): +class FactGetHostVersionTest(BaseFactTest): def setUp(self): super(FactGetHostVersionTest, self).setUp() - create_host_document() + self.create_host_document(TEST_FACT_DATA) self.t1 = datetime.now().replace(second=1, microsecond=0) self.t2 = datetime.now().replace(second=2, microsecond=0) @@ -137,10 +136,10 @@ class FactGetHostVersionTest(BaseTest, MongoDBRequired): fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module']) self.assertIsNone(fact) -class FactGetHostTimeline(BaseTest, MongoDBRequired): +class FactGetHostTimelineTest(BaseFactTest): def setUp(self): - super(FactGetHostTimeline, self).setUp() - create_host_document() + super(FactGetHostTimelineTest, self).setUp() + self.create_host_document(TEST_FACT_DATA) self.scans = 20 self.timestamps = create_fact_scans(self.scans) @@ -151,4 +150,3 @@ class FactGetHostTimeline(BaseTest, MongoDBRequired): self.assertEqual(len(timestamps), len(self.timestamps)) for i in range(0, self.scans): self.assertEqual(timestamps[i], self.timestamps[i]) -