From 0c7638bdd7b9f320ad507b0d561c6bfba74cae5b Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Tue, 9 Jun 2015 11:32:10 -0400 Subject: [PATCH] associate scan runs with a particular inventory host --- awx/api/views.py | 4 ++-- awx/fact/models/fact.py | 24 ++++++++----------- awx/fact/tests/base.py | 10 +++++++- awx/fact/tests/models/fact/fact_simple.py | 19 ++++++++------- awx/fact/tests/models/fact/fact_transform.py | 6 ++--- .../commands/run_fact_cache_receiver.py | 7 +++--- awx/main/tests/fact/fact_api.py | 5 ++++ awx/plugins/fact_caching/tower.py | 9 ++++++- 8 files changed, 51 insertions(+), 33 deletions(-) diff --git a/awx/api/views.py b/awx/api/views.py index 30bd034619..e501440838 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -1162,7 +1162,7 @@ class HostFactVersionsList(MongoListAPIView): self.check_parent_access(host) try: - fact_host = FactHost.objects.get(hostname=host.name) + fact_host = FactHost.objects.get(hostname=host.name, inventory_id=host.inventory.pk) except FactHost.DoesNotExist: return None except mongoengine.ConnectionError: @@ -1234,7 +1234,7 @@ class HostFactCompareView(MongoAPIView): datetime_actual = dateutil.parser.parse(datetime_spec) if datetime_spec is not None else now() host_obj = self.get_parent_object() - fact_entry = Fact.get_host_version(host_obj.name, datetime_actual, module_spec) + fact_entry = Fact.get_host_version(host_obj.name, host_obj.inventory.pk, datetime_actual, module_spec) host_data = FactSerializer(fact_entry).data if fact_entry is not None else {} return Response(host_data) diff --git a/awx/fact/models/fact.py b/awx/fact/models/fact.py index 2ec6a8e6a0..4117d0f1ac 100644 --- a/awx/fact/models/fact.py +++ b/awx/fact/models/fact.py @@ -2,7 +2,7 @@ # All Rights Reserved from mongoengine.base import BaseField -from mongoengine import Document, DateTimeField, ReferenceField, StringField +from mongoengine import Document, DateTimeField, ReferenceField, StringField, IntField from awx.fact.utils.dbtransform import KeyTransform key_transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')]) @@ -22,22 +22,17 @@ class TransformField(BaseField): class FactHost(Document): hostname = StringField(max_length=100, required=True, unique=True) + inventory_id = IntField(required=True) # TODO: Consider using hashed index on hostname. django-mongo may not support this but # executing raw js will meta = { 'indexes': [ - 'hostname' + 'hostname', + 'inventory_id' ] } - @staticmethod - def get_host_id(hostname): - host = FactHost.objects.get(hostname=hostname) - if host: - return host.id - return None - class Fact(Document): timestamp = DateTimeField(required=True) host = ReferenceField(FactHost, required=True) @@ -49,7 +44,7 @@ class Fact(Document): meta = { 'indexes': [ '-timestamp', - 'host' + 'host', ] } @@ -65,9 +60,9 @@ class Fact(Document): # If module not specified then filter query may return more than 1 result. # Thus, the resulting facts must somehow be unioned/concated/ or kept as an array. @staticmethod - def get_host_version(hostname, timestamp, module): + def get_host_version(hostname, inventory_id, timestamp, module): try: - host = FactHost.objects.get(hostname=hostname) + host = FactHost.objects.get(hostname=hostname, inventory_id=inventory_id) except FactHost.DoesNotExist: return None @@ -86,9 +81,9 @@ class Fact(Document): return None @staticmethod - def get_host_timeline(hostname, module): + def get_host_timeline(hostname, inventory_id, module): try: - host = FactHost.objects.get(hostname=hostname) + host = FactHost.objects.get(hostname=hostname, inventory_id=inventory_id) except FactHost.DoesNotExist: return None @@ -99,6 +94,7 @@ class Fact(Document): return FactVersion.objects.filter(**kv).order_by("-timestamp").values_list('timestamp') + # FIXME: single facts no longer works with the addition of the inventory_id field to the FactHost document @staticmethod def get_single_facts(hostnames, fact_key, fact_value, timestamp, module): kv = { diff --git a/awx/fact/tests/base.py b/awx/fact/tests/base.py index aaa90cf376..ff5126bdfb 100644 --- a/awx/fact/tests/base.py +++ b/awx/fact/tests/base.py @@ -93,11 +93,13 @@ class BaseFactTestMixin(MongoDBRequired): class BaseFactTest(BaseFactTestMixin, MongoDBRequired): pass +# TODO: for now, we relate all hosts to a single inventory class FactScanBuilder(object): def __init__(self): self.facts_data = {} self.hostname_data = [] + self.inventory_id = 1 self.host_objs = [] self.fact_objs = [] @@ -124,7 +126,7 @@ class FactScanBuilder(object): if len(self.hostname_data) == 0: self.hostname_data = ['hostname_%s' % i for i in range(0, host_count)] - self.host_objs = [FactHost(hostname=hostname).save() for hostname in self.hostname_data] + self.host_objs = [FactHost(hostname=hostname, inventory_id=self.inventory_id).save() for hostname in self.hostname_data] for i in range(0, scan_count): scan = {} @@ -186,6 +188,12 @@ class FactScanBuilder(object): return [self.host_objs[i].hostname for i in range(index_start, index_end)] + def get_inventory_id(self): + return self.inventory_id + + def set_inventory_id(self, inventory_id): + self.inventory_id = inventory_id + def get_host(self, index): return self.host_objs[index] diff --git a/awx/fact/tests/models/fact/fact_simple.py b/awx/fact/tests/models/fact/fact_simple.py index 8914d9f2f4..d8aa4c4927 100644 --- a/awx/fact/tests/models/fact/fact_simple.py +++ b/awx/fact/tests/models/fact/fact_simple.py @@ -16,19 +16,20 @@ __all__ = ['FactHostTest', 'FactTest', 'FactGetHostVersionTest', 'FactGetHostTim class FactHostTest(BaseFactTest): def test_create_host(self): - host = FactHost(hostname='hosty') + host = FactHost(hostname='hosty', inventory_id=1) host.save() - host = FactHost.objects.get(hostname='hosty') + host = FactHost.objects.get(hostname='hosty', inventory_id=1) self.assertIsNotNone(host, "Host added but not found") self.assertEqual('hosty', host.hostname, "Gotten record hostname does not match expected hostname") + self.assertEqual(1, host.inventory_id, "Gotten record inventory_id does not match expected inventory_id") # Ensure an error is raised for .get() that doesn't match a record. def test_get_host_id_no_result(self): - host = FactHost(hostname='hosty') + host = FactHost(hostname='hosty', inventory_id=1) host.save() - self.assertRaises(FactHost.DoesNotExist, FactHost.objects.get, hostname='doesnotexist') + self.assertRaises(FactHost.DoesNotExist, FactHost.objects.get, hostname='doesnotexist', inventory_id=1) class FactTest(BaseFactTest): def setUp(self): @@ -36,7 +37,7 @@ class FactTest(BaseFactTest): def test_add_fact(self): timestamp = now().replace(microsecond=0) - host = FactHost(hostname="hosty").save() + host = FactHost(hostname="hosty", inventory_id=1).save() (f_obj, v_obj) = Fact.add_fact(host=host, timestamp=timestamp, module='packages', fact=TEST_FACT_PACKAGES) f = Fact.objects.get(id=f_obj.id) v = FactVersion.objects.get(id=v_obj.id) @@ -65,20 +66,20 @@ class FactGetHostVersionTest(BaseFactTest): def test_get_host_version_exact_timestamp(self): fact_known = self.builder.get_scan(0, 'packages')[0] - fact = Fact.get_host_version(hostname=self.builder.get_hostname(0), timestamp=self.builder.get_timestamp(0), module='packages') + fact = Fact.get_host_version(hostname=self.builder.get_hostname(0), inventory_id=self.builder.get_inventory_id(), timestamp=self.builder.get_timestamp(0), module='packages') self.assertIsNotNone(fact) self.assertEqual(fact_known, fact) def test_get_host_version_lte_timestamp(self): timestamp = self.builder.get_timestamp(0) + relativedelta(days=1) fact_known = self.builder.get_scan(0, 'packages')[0] - fact = Fact.get_host_version(hostname=self.builder.get_hostname(0), timestamp=timestamp, module='packages') + fact = Fact.get_host_version(hostname=self.builder.get_hostname(0), inventory_id=self.builder.get_inventory_id(), timestamp=timestamp, module='packages') self.assertIsNotNone(fact) self.assertEqual(fact_known, fact) def test_get_host_version_none(self): timestamp = self.builder.get_timestamp(0) - relativedelta(years=20) - fact = Fact.get_host_version(hostname=self.builder.get_hostname(0), timestamp=timestamp, module='packages') + fact = Fact.get_host_version(hostname=self.builder.get_hostname(0), inventory_id=self.builder.get_inventory_id(), timestamp=timestamp, module='packages') self.assertIsNone(fact) class FactGetHostTimelineTest(BaseFactTest): @@ -89,7 +90,7 @@ class FactGetHostTimelineTest(BaseFactTest): self.builder.build(scan_count=20, host_count=1) def test_get_host_timeline_ok(self): - timestamps = Fact.get_host_timeline(hostname=self.builder.get_hostname(0), module='packages') + timestamps = Fact.get_host_timeline(hostname=self.builder.get_hostname(0), inventory_id=self.builder.get_inventory_id(), module='packages') self.assertIsNotNone(timestamps) self.assertEqual(len(timestamps), self.builder.get_scan_count()) for i in range(0, self.builder.get_scan_count()): diff --git a/awx/fact/tests/models/fact/fact_transform.py b/awx/fact/tests/models/fact/fact_transform.py index 1f4d95f128..79013cb0cd 100644 --- a/awx/fact/tests/models/fact/fact_transform.py +++ b/awx/fact/tests/models/fact/fact_transform.py @@ -62,12 +62,12 @@ class FactTransformTest(BaseFactTest): self.timestamp = datetime.now().replace(microsecond=0) def setup_create_fact_dot(self): - self.host = FactHost(hostname='hosty').save() + self.host = FactHost(hostname='hosty', inventory_id=1).save() self.f = Fact(timestamp=self.timestamp, module='packages', fact=TEST_FACT_PACKAGES_WITH_DOTS, host=self.host) self.f.save() def setup_create_fact_dollar(self): - self.host = FactHost(hostname='hosty').save() + self.host = FactHost(hostname='hosty', inventory_id=1).save() self.f = Fact(timestamp=self.timestamp, module='packages', fact=TEST_FACT_PACKAGES_WITH_DOLLARS, host=self.host) self.f.save() @@ -85,7 +85,7 @@ class FactTransformTest(BaseFactTest): def test_fact_with_dot_serialized_pymongo(self): #self.setup_create_fact_dot() - host = FactHost(hostname='hosty').save() + host = FactHost(hostname='hosty', inventory_id=1).save() f = self.db['fact'].insert({ 'hostname': 'hosty', 'fact': TEST_FACT_PACKAGES_WITH_DOTS, diff --git a/awx/main/management/commands/run_fact_cache_receiver.py b/awx/main/management/commands/run_fact_cache_receiver.py index ec16f0a4f2..bc6b8c6db5 100644 --- a/awx/main/management/commands/run_fact_cache_receiver.py +++ b/awx/main/management/commands/run_fact_cache_receiver.py @@ -34,6 +34,7 @@ class FactCacheReceiver(object): def process_fact_message(self, message): hostname = message['host'] + inventory_id = message['inventory_id'] facts_data = message['facts'] date_key = message['date_key'] @@ -43,12 +44,12 @@ class FactCacheReceiver(object): return try: - host = FactHost.objects.get(hostname=hostname) + host = FactHost.objects.get(hostname=hostname, inventory_id=inventory_id) except FactHost.DoesNotExist: - host = FactHost(hostname=hostname) + host = FactHost(hostname=hostname, inventory_id=inventory_id) host.save() except FactHost.MultipleObjectsReturned: - query = "db['fact_host'].find(hostname=%s)" % hostname + query = "db['fact_host'].find(hostname=%s, inventory_id=%s)" % (hostname, inventory_id) logger.warn('Database inconsistent. Multiple FactHost "%s" exist. Try the query %s to find the records.' % (hostname, query)) return except Exception, e: diff --git a/awx/main/tests/fact/fact_api.py b/awx/main/tests/fact/fact_api.py index 343eede24d..4382331dfa 100644 --- a/awx/main/tests/fact/fact_api.py +++ b/awx/main/tests/fact/fact_api.py @@ -2,6 +2,7 @@ # All Rights Reserved # Python +import unittest # Django from django.core.urlresolvers import reverse @@ -30,6 +31,7 @@ class FactApiBaseTest(BaseLiveServerTest, BaseFactTestMixin): def setup_facts(self, scan_count): self.builder = FactScanBuilder() + self.builder.set_inventory_id(self.inventory.pk) self.builder.add_fact('ansible', TEST_FACT_ANSIBLE) self.builder.add_fact('packages', TEST_FACT_PACKAGES) self.builder.add_fact('services', TEST_FACT_SERVICES) @@ -140,6 +142,7 @@ class FactViewApiTest(FactApiBaseTest): 'module': fact_obj.module, 'host': { 'hostname': fact_obj.host.hostname, + 'inventory_id': fact_obj.host.inventory_id, 'id': str(fact_obj.host.id) }, 'fact': fact_obj.fact @@ -180,6 +183,8 @@ class FactViewApiTest(FactApiBaseTest): self.get_fact(Fact.objects.filter(host=self.fact_host, module='ansible', timestamp__lte=ts).order_by('-timestamp')[0], dict(datetime=ts)) + +@unittest.skip("single fact query needs to be updated to use inventory_id attribute on host document") class SingleFactApiTest(FactApiBaseTest): def setUp(self): super(SingleFactApiTest, self).setUp() diff --git a/awx/plugins/fact_caching/tower.py b/awx/plugins/fact_caching/tower.py index 8e80280afd..6210118712 100755 --- a/awx/plugins/fact_caching/tower.py +++ b/awx/plugins/fact_caching/tower.py @@ -30,6 +30,7 @@ # POSSIBILITY OF SUCH DAMAGE. import sys +import os import time import datetime from copy import deepcopy @@ -111,8 +112,14 @@ class CacheModule(BaseCacheModule): self._cache_prev = deepcopy(self._cache) self._cache[key] = value + packet = { + 'host': key, + 'inventory_id': os.environ['INVENTORY_ID'], + 'facts': facts, + 'date_key': self.date_key, + } # Emit fact data to tower for processing - self.socket.send_json(dict(host=key, facts=facts, date_key=self.date_key)) + self.socket.send_json(packet) self.socket.recv() def keys(self):