From f3739ec283fa59a5341dfdf240a88df6943af1c7 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Tue, 5 May 2015 10:53:49 -0400 Subject: [PATCH] single fact now behaves like a match query --- awx/fact/models/fact.py | 23 ++-- .../models/fact/fact_get_single_facts.py | 107 ++++++++++++------ 2 files changed, 87 insertions(+), 43 deletions(-) diff --git a/awx/fact/models/fact.py b/awx/fact/models/fact.py index e3705ee493..34e44256ce 100644 --- a/awx/fact/models/fact.py +++ b/awx/fact/models/fact.py @@ -100,7 +100,7 @@ class Fact(Document): return FactVersion.objects.filter(**kv).values_list('timestamp') @staticmethod - def get_single_facts(hostnames, fact_key, timestamp, module): + def get_single_facts(hostnames, fact_key, fact_value, timestamp, module): host_ids = FactHost.objects.filter(hostname__in=hostnames).values_list('id') if not host_ids or len(host_ids) == 0: return None @@ -118,14 +118,23 @@ class Fact(Document): # This is not a logic problem, but a performance problem. fact_ids = [fact.id for fact in facts] - project = { - '$project': { - 'host': 1, - 'fact.%s' % fact_key: 1, + kv = { + 'fact.%s' % fact_key : fact_value, + '_id': { + '$in': fact_ids } } - facts = Fact.objects.filter(id__in=fact_ids).aggregate(project) - return facts + fields = { + 'fact.%s.$' % fact_key : 1, + } + facts = Fact._get_collection().find(kv, fields) + #fact_objs = [Fact(**f) for f in facts] + # Translate pymongo python structure to mongoengine Fact object + fact_objs = [] + for f in facts: + f['id'] = f.pop('_id') + fact_objs.append(Fact(**f)) + return fact_objs class FactVersion(Document): diff --git a/awx/fact/tests/models/fact/fact_get_single_facts.py b/awx/fact/tests/models/fact/fact_get_single_facts.py index 9329f0d653..7322ee2853 100644 --- a/awx/fact/tests/models/fact/fact_get_single_facts.py +++ b/awx/fact/tests/models/fact/fact_get_single_facts.py @@ -14,49 +14,60 @@ from .base import BaseFactTest __all__ = ['FactGetSingleFactsTest'] +TEST_FACT_PACKAGES = [ + { + "name": "accountsservice", + "architecture": "amd64", + "name": "accountsservice", + "source": "apt", + "version": "0.6.35-0ubuntu7.1" + }, + { + "name": "acpid", + "architecture": "amd64", + "name": "acpid", + "source": "apt", + "version": "1:2.0.21-1ubuntu2" + }, + { + "name": "adduser", + "architecture": "all", + "name": "adduser", + "source": "apt", + "version": "3.113+nmu3ubuntu3" + }, +] + TEST_FACT_DATA = { + 'hostname': 'hostname_%d', + 'add_fact_data': { + 'timestamp': datetime.now(), + 'host': None, + 'module': 'packages', + 'fact': TEST_FACT_PACKAGES, + } +} + +TEST_FACT_NESTED_DATA = { 'hostname': 'hostname_%d', 'add_fact_data': { 'timestamp': datetime.now(), 'host': None, 'module': 'packages', 'fact': { - "accountsservice": [ - { - "architecture": "amd64", - "name": "accountsservice", - "source": "apt", - "version": "0.6.35-0ubuntu7.1" - } - ], - "acpid": [ - { - "architecture": "amd64", - "name": "acpid", - "source": "apt", - "version": "1:2.0.21-1ubuntu2" - } - ], - "adduser": [ - { - "architecture": "all", - "name": "adduser", - "source": "apt", - "version": "3.113+nmu3ubuntu3" - } - ], + 'nested': TEST_FACT_PACKAGES }, } } class FactGetSingleFactsTest(BaseFactTest): - def create_fact_scans_unique_hosts(self, host_count): + def create_fact_scans_unique_hosts(self, data, host_count): self.fact_data = [] self.fact_objs = [] self.hostnames = [] for i in range(1, host_count + 1): - fact_data = deepcopy(TEST_FACT_DATA) + fact_data = deepcopy(data) fact_data['hostname'] = fact_data['hostname'] % (i) fact_data['add_fact_data']['timestamp'] = datetime.now().replace(year=2015 - i) BaseFactTest.normalize_timestamp(fact_data) @@ -68,18 +79,26 @@ class FactGetSingleFactsTest(BaseFactTest): self.fact_objs.append(fact_obj) self.hostnames.append(fact_data['hostname']) - def setUp(self): - super(FactGetSingleFactsTest, self).setUp() + def setup_test_fact_data(self): self.host_count = 20 - self.create_fact_scans_unique_hosts(self.host_count) + self.create_fact_scans_unique_hosts(TEST_FACT_DATA, self.host_count) + + def setup_test_fact_nested_data(self): + self.host_count = 20 + self.create_fact_scans_unique_hosts(TEST_FACT_NESTED_DATA, self.host_count) def check_query_results(self, facts_known, facts): - # Transpose facts to a dict with key _id + # Ensure only 'acpid' is returned + for fact in facts: + self.assertEqual(len(fact.fact), 1) + self.assertEqual(fact.fact[0]['name'], 'acpid') + + # Transpose facts to a dict with key id count = 0 facts_dict = {} for fact in facts: count += 1 - facts_dict[fact['_id']] = fact + facts_dict[fact.id] = fact self.assertEqual(count, len(facts_known)) # For each fact that we put into the database on setup, @@ -87,20 +106,36 @@ class FactGetSingleFactsTest(BaseFactTest): for fact_known in facts_known: key = fact_known.id self.assertIn(key, facts_dict) - self.assertEqual(facts_dict[key]['fact']['acpid'], fact_known.fact['acpid']) - self.assertEqual(facts_dict[key]['host'], fact_known.host.id) + self.assertEqual(len(facts_dict[key].fact), 1) + + def check_query_results_nested(self, facts): + for fact in facts: + self.assertEqual(len(fact.fact), 1) + self.assertEqual(fact.fact['nested'][0]['name'], 'acpid') def test_get_single_facts_ok(self): + self.setup_test_fact_data() + timestamp = datetime.now().replace(year=2016) - facts = Fact.get_single_facts(self.hostnames, 'acpid', timestamp, 'packages') + facts = Fact.get_single_facts(self.hostnames, 'name', 'acpid', timestamp, 'packages') self.assertIsNotNone(facts) self.check_query_results(self.fact_objs, facts) def test_get_single_facts_subset_by_timestamp(self): + self.setup_test_fact_data() + timestamp = datetime.now().replace(year=2010) - facts = Fact.get_single_facts(self.hostnames, 'acpid', timestamp, 'packages') + facts = Fact.get_single_facts(self.hostnames, 'name', 'acpid', timestamp, 'packages') self.assertIsNotNone(facts) self.check_query_results(self.fact_objs[4:], facts) - \ No newline at end of file + + def test_get_single_facts_nested(self): + self.setup_test_fact_nested_data() + + timestamp = datetime.now().replace(year=2016) + facts = Fact.get_single_facts(self.hostnames, 'nested.name', 'acpid', timestamp, 'packages') + self.assertIsNotNone(facts) + + self.check_query_results_nested(facts)