diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 86fa7998e5..ec913423a8 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -36,7 +36,7 @@ from polymorphic import PolymorphicModel # AWX from awx.main.constants import SCHEDULEABLE_PROVIDERS from awx.main.models import * # noqa -from awx.main.utils import get_type_for_model, get_model_for_type +from awx.main.utils import get_type_for_model, get_model_for_type, build_url, timestamp_apiformat from awx.main.redact import REPLACE_STR from awx.fact.models import * # noqa @@ -2017,10 +2017,23 @@ class AuthTokenSerializer(serializers.Serializer): class FactVersionSerializer(MongoEngineModelSerializer): + related = serializers.SerializerMethodField('get_related') class Meta: model = FactVersion - fields = ('module', 'timestamp',) + fields = ('related', 'module', 'timestamp',) + + def get_related(self, obj): + host_obj = self.context.get('host_obj') + res = {} + params = { + 'datetime': timestamp_apiformat(obj.timestamp), + 'module': obj.module, + } + res.update(dict( + fact_view = build_url('api:host_fact_compare_view', args=(host_obj.pk,), get=params), + )) + return res class FactSerializer(MongoEngineModelSerializer): diff --git a/awx/api/views.py b/awx/api/views.py index e81f8845d0..9380c006d8 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -1037,12 +1037,17 @@ class HostFactVersionsList(MongoListAPIView): if from_spec is not None: from_actual = dateutil.parser.parse(from_spec) kv['timestamp__gt'] = from_actual - if from_spec is not None and to_spec is not None: + if to_spec is not None: to_actual = dateutil.parser.parse(to_spec) kv['timestamp__lte'] = to_actual return FactVersion.objects.filter(**kv).order_by("-timestamp") + def list(self, *args, **kwargs): + queryset = self.get_queryset() or [] + serializer = FactVersionSerializer(queryset, many=True, context=dict(host_obj=self.get_parent_object())) + return Response(dict(results=serializer.data)) + class HostSingleFactView(MongoAPIView): model = Fact @@ -1062,7 +1067,7 @@ class HostSingleFactView(MongoAPIView): datetime_actual = dateutil.parser.parse(datetime_spec) if datetime_spec is not None else now() host_obj = self.get_parent_object() fact_data = Fact.get_single_facts([host_obj.name], fact_key, fact_value, datetime_actual, module_spec) - return Response(FactSerializer(fact_data).data if fact_data is not None else {}) + return Response(FactSerializer(fact_data, context=dict(host_obj=host_obj)).data if fact_data is not None else {}) class HostFactCompareView(MongoAPIView): @@ -1082,7 +1087,6 @@ class HostFactCompareView(MongoAPIView): return Response(host_data) - class GroupList(ListCreateAPIView): model = Group diff --git a/awx/fact/__init__.py b/awx/fact/__init__.py index f9d5796ca2..cc9b260832 100644 --- a/awx/fact/__init__.py +++ b/awx/fact/__init__.py @@ -14,7 +14,7 @@ logger = logging.getLogger('awx.fact') # Connect to Mongo try: - connect(settings.MONGO_DB) + connect(settings.MONGO_DB, tz_aware=settings.USE_TZ) register_key_transform(get_db()) except ConnectionError: logger.warn('Failed to establish connect to MongoDB "%s"' % (settings.MONGO_DB)) diff --git a/awx/fact/tests/__init__.py b/awx/fact/tests/__init__.py index d7187f3928..d276b01707 100644 --- a/awx/fact/tests/__init__.py +++ b/awx/fact/tests/__init__.py @@ -5,3 +5,4 @@ from __future__ import absolute_import from .models import * # noqa from .utils import * # noqa +from .base import * # noqa diff --git a/awx/fact/tests/base.py b/awx/fact/tests/base.py new file mode 100644 index 0000000000..09c9e32ae0 --- /dev/null +++ b/awx/fact/tests/base.py @@ -0,0 +1,202 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python +from __future__ import absolute_import +from copy import deepcopy +from datetime import datetime +from django.utils.timezone import now + +# Django +from django.conf import settings +import django + +# MongoEngine +from mongoengine.connection import get_db, ConnectionError + +# AWX +from awx.fact.models.fact import * # noqa + +TEST_FACT_ANSIBLE = { + "ansible_swapfree_mb" : 4092, + "ansible_default_ipv6" : { + + }, + "ansible_distribution_release" : "trusty", + "ansible_system_vendor" : "innotek GmbH", + "ansible_os_family" : "Debian", + "ansible_all_ipv4_addresses" : [ + "192.168.1.145" + ], + "ansible_lsb" : { + "release" : "14.04", + "major_release" : "14", + "codename" : "trusty", + "id" : "Ubuntu", + "description" : "Ubuntu 14.04.2 LTS" + }, +} + +TEST_FACT_PACKAGES = [ + { + "name": "accountsservice", + "architecture": "amd64", + "source": "apt", + "version": "0.6.35-0ubuntu7.1" + }, + { + "name": "acpid", + "architecture": "amd64", + "source": "apt", + "version": "1:2.0.21-1ubuntu2" + }, + { + "name": "adduser", + "architecture": "all", + "source": "apt", + "version": "3.113+nmu3ubuntu3" + }, +] + +TEST_FACT_SERVICES = [ + { + "source" : "upstart", + "state" : "waiting", + "name" : "ureadahead-other", + "goal" : "stop" + }, + { + "source" : "upstart", + "state" : "running", + "name" : "apport", + "goal" : "start" + }, + { + "source" : "upstart", + "state" : "waiting", + "name" : "console-setup", + "goal" : "stop" + }, +] + + +class MongoDBRequired(django.test.TestCase): + def setUp(self): + # Drop mongo database + try: + self.db = get_db() + self.db.connection.drop_database(settings.MONGO_DB) + except ConnectionError: + self.skipTest('MongoDB connection failed') + +class BaseFactTestMixin(MongoDBRequired): + pass + +class BaseFactTest(BaseFactTestMixin, MongoDBRequired): + pass + +class FactScanBuilder(object): + + def __init__(self): + self.facts_data = {} + self.hostname_data = [] + + self.host_objs = [] + self.fact_objs = [] + self.version_objs = [] + self.timestamps = [] + + def add_fact(self, module, facts): + self.facts_data[module] = facts + + def add_hostname(self, hostname): + self.hostname_data.append(hostname) + + def build(self, scan_count, host_count): + if len(self.facts_data) == 0: + raise RuntimeError("No fact data to build populate scans. call add_fact()") + if (len(self.hostname_data) > 0 and len(self.hostname_data) != host_count): + raise RuntimeError("Registered number of hostnames %d does not match host_count %d" % (len(self.hostname_data), host_count)) + + if len(self.hostname_data) == 0: + self.hostname_data = ['hostname_%s' % i for i in range(0, host_count)] + + self.host_objs = [FactHost(hostname=hostname).save() for hostname in self.hostname_data] + + for i in range(0, scan_count): + scan = {} + scan_version = {} + timestamp = now().replace(year=2015 - i, microsecond=0) + for module in self.facts_data: + fact_objs = [] + version_objs = [] + for host in self.host_objs: + (fact_obj, version_obj) = Fact.add_fact(timestamp=timestamp, + host=host, + module=module, + fact=self.facts_data[module]) + fact_objs.append(fact_obj) + version_objs.append(version_obj) + scan[module] = fact_objs + scan_version[module] = version_objs + self.fact_objs.append(scan) + self.version_objs.append(scan_version) + self.timestamps.append(timestamp) + + + def get_scan(self, index, module=None): + res = None + res = self.fact_objs[index] + if module: + res = res[module] + return res + + def get_scans(self, index_start=None, index_end=None): + if index_start is None: + index_start = 0 + if index_end is None: + index_end = len(self.fact_objs) + return self.fact_objs[index_start:index_end] + + def get_scan_version(self, index, module=None): + res = None + res = self.version_objs[index] + if module: + res = res[module] + return res + + def get_scan_versions(self, index_start=None, index_end=None): + if index_start is None: + index_start = 0 + if index_end is None: + index_end = len(self.version_objs) + return self.version_objs[index_start:index_end] + + def get_hostname(self, index): + return self.host_objs[index].hostname + + def get_hostnames(self, index_start=None, index_end=None): + if index_start is None: + index_start = 0 + if index_end is None: + index_end = len(self.host_objs) + + return [self.host_objs[i].hostname for i in range(index_start, index_end)] + + + def get_scan_count(self): + return len(self.fact_objs) + + def get_host_count(self): + return len(self.host_objs) + + def get_timestamp(self, index): + return self.timestamps[index] + + def get_timestamps(self, index_start=None, index_end=None): + if not index_start: + index_start = 0 + if not index_end: + len(self.timestamps) + return self.timestamps[index_start:index_end] + diff --git a/awx/fact/tests/models/fact/base.py b/awx/fact/tests/models/fact/base.py deleted file mode 100644 index 28544dab4e..0000000000 --- a/awx/fact/tests/models/fact/base.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2015 Ansible, Inc. -# All Rights Reserved - -# Python -from __future__ import absolute_import -from awx.main.tests.base import BaseTest, MongoDBRequired -from copy import deepcopy -from datetime import datetime - -# AWX -from awx.fact.models.fact import * # noqa - -''' -Helper functions (i.e. create_host_document) expect the structure: -{ - 'hostname': 'hostname1', - 'add_fact_data': { - 'timestamp': datetime.now(), - 'host': None, - 'module': 'packages', - 'fact': ... - } -} -''' -class BaseFactTest(BaseTest, MongoDBRequired): - - @staticmethod - def _normalize_timestamp(timestamp): - return timestamp.replace(microsecond=0) - - @staticmethod - def normalize_timestamp(data): - data['add_fact_data']['timestamp'] = BaseFactTest._normalize_timestamp(data['add_fact_data']['timestamp']) - - def create_host_document(self, data): - data['add_fact_data']['host'] = FactHost(hostname=data['hostname']).save() - - def create_fact_scans(self, data, host_count=1, scan_count=1): - timestamps = [] - self.fact_data = [] - self.fact_objs = [] - self.hostnames = [FactHost(hostname='%s_%s' % (data['hostname'], i)).save() for i in range(0, host_count)] - for i in range(0, scan_count): - self.fact_data.append([]) - self.fact_objs.append([]) - for j in range(0, host_count): - data = deepcopy(data) - t = datetime.now().replace(year=2015 - i, microsecond=0) - data['add_fact_data']['timestamp'] = t - data['add_fact_data']['host'] = self.hostnames[j] - (f, v) = Fact.add_fact(**data['add_fact_data']) - timestamps.append(t) - - self.fact_data[i].append(data) - self.fact_objs[i].append(f) - - return timestamps diff --git a/awx/fact/tests/models/fact/fact_get_single_facts.py b/awx/fact/tests/models/fact/fact_get_single_facts.py index 9b5f25cb8b..51607c94af 100644 --- a/awx/fact/tests/models/fact/fact_get_single_facts.py +++ b/awx/fact/tests/models/fact/fact_get_single_facts.py @@ -9,61 +9,17 @@ from datetime import datetime # AWX from awx.fact.models.fact import * # noqa -from .base import BaseFactTest +from awx.fact.tests.base import BaseFactTest, FactScanBuilder, TEST_FACT_PACKAGES __all__ = ['FactGetSingleFactsTest', 'FactGetSingleFactsMultipleScansTest',] -TEST_FACT_PACKAGES = [ - { - "name": "accountsservice", - "architecture": "amd64", - "source": "apt", - "version": "0.6.35-0ubuntu7.1" - }, - { - "name": "acpid", - "architecture": "amd64", - "source": "apt", - "version": "1:2.0.21-1ubuntu2" - }, - { - "name": "adduser", - "architecture": "all", - "source": "apt", - "version": "3.113+nmu3ubuntu3" - }, -] - -TEST_FACT_DATA = { - 'hostname': 'hostname_%d', - 'add_fact_data': { - 'timestamp': datetime.now(), - 'host': None, - 'module': 'packages', - 'fact': TEST_FACT_PACKAGES, - } -} - -TEST_FACT_NESTED_DATA = { - 'hostname': 'hostname_%d', - 'add_fact_data': { - 'timestamp': datetime.now(), - 'host': None, - 'module': 'packages', - 'fact': { - 'nested': TEST_FACT_PACKAGES - }, - } -} - - class FactGetSingleFactsTest(BaseFactTest): def setUp(self): super(FactGetSingleFactsTest, self).setUp() - self.host_count = 20 - self.timestamp = datetime.now().replace(year=2016) - self.create_fact_scans(TEST_FACT_DATA, self.host_count, scan_count=1) - self.hosts = [self.hostnames[i].hostname for i in range(0, self.host_count)] + self.builder = FactScanBuilder() + self.builder.add_fact('packages', TEST_FACT_PACKAGES) + self.builder.add_fact('nested', TEST_FACT_PACKAGES) + self.builder.build(scan_count=1, host_count=20) def check_query_results(self, facts_known, facts): self.assertIsNotNone(facts) @@ -95,50 +51,47 @@ class FactGetSingleFactsTest(BaseFactTest): self.assertEqual(fact.fact['nested'][0]['name'], 'acpid') def test_single_host(self): - self.hosts = [self.hostnames[i].hostname for i in range(0, 1)] - facts = Fact.get_single_facts(self.hosts, 'name', 'acpid', self.timestamp, 'packages') + facts = Fact.get_single_facts(self.builder.get_hostnames(0, 1), 'name', 'acpid', self.builder.get_timestamp(0), 'packages') - self.check_query_results(self.fact_objs[0][:1], facts) + self.check_query_results(self.builder.get_scan(0, 'packages')[:1], facts) def test_all(self): - facts = Fact.get_single_facts(self.hosts, 'name', 'acpid', self.timestamp, 'packages') + facts = Fact.get_single_facts(self.builder.get_hostnames(), 'name', 'acpid', self.builder.get_timestamp(0), 'packages') - self.check_query_results(self.fact_objs[0], facts) + self.check_query_results(self.builder.get_scan(0, 'packages'), facts) def test_subset_hosts(self): - self.hosts = [self.hostnames[i].hostname for i in range(0, (self.host_count / 2))] - facts = Fact.get_single_facts(self.hosts, 'name', 'acpid', self.timestamp, 'packages') + host_count = (self.builder.get_host_count() / 2) + facts = Fact.get_single_facts(self.builder.get_hostnames(0, host_count), 'name', 'acpid', self.builder.get_timestamp(0), 'packages') - self.check_query_results(self.fact_objs[0][:(self.host_count / 2)], facts) + self.check_query_results(self.builder.get_scan(0, 'packages')[:host_count], facts) def test_get_single_facts_nested(self): - facts = Fact.get_single_facts(self.hosts, 'nested.name', 'acpid', self.timestamp, 'packages') + facts = Fact.get_single_facts(self.builder.get_hostnames(), 'nested.name', 'acpid', self.builder.get_timestamp(0), 'packages') self.check_query_results_nested(facts) class FactGetSingleFactsMultipleScansTest(BaseFactTest): def setUp(self): super(FactGetSingleFactsMultipleScansTest, self).setUp() - self.create_fact_scans(TEST_FACT_DATA, host_count=10, scan_count=10) + self.builder = FactScanBuilder() + self.builder.add_fact('packages', TEST_FACT_PACKAGES) + self.builder.build(scan_count=10, host_count=10) def test_1_host(self): - timestamp = datetime.now().replace(year=2016) - facts = Fact.get_single_facts([self.hostnames[0].hostname], 'name', 'acpid', timestamp, 'packages') + facts = Fact.get_single_facts(self.builder.get_hostnames(0, 1), 'name', 'acpid', self.builder.get_timestamp(0), 'packages') self.assertEqual(len(facts), 1) - self.assertEqual(facts[0], self.fact_objs[0][0]) + self.assertEqual(facts[0], self.builder.get_scan(0, 'packages')[0]) def test_multiple_hosts(self): - timestamp = datetime.now().replace(year=2016) - hosts = [self.hostnames[i].hostname for i in range(0, 3)] - facts = Fact.get_single_facts(hosts, 'name', 'acpid', timestamp, 'packages') + facts = Fact.get_single_facts(self.builder.get_hostnames(0, 3), 'name', 'acpid', self.builder.get_timestamp(0), 'packages') self.assertEqual(len(facts), 3) for i, fact in enumerate(facts): - self.assertEqual(fact, self.fact_objs[0][i]) + self.assertEqual(fact, self.builder.get_scan(0, 'packages')[i]) def test_middle_of_timeline(self): - timestamp = datetime.now().replace(year=2013) - hosts = [self.hostnames[i].hostname for i in range(0, 3)] - facts = Fact.get_single_facts(hosts, 'name', 'acpid', timestamp, 'packages') + facts = Fact.get_single_facts(self.builder.get_hostnames(0, 3), 'name', 'acpid', self.builder.get_timestamp(4), 'packages') self.assertEqual(len(facts), 3) for i, fact in enumerate(facts): - self.assertEqual(fact, self.fact_objs[2][i]) + self.assertEqual(fact, self.builder.get_scan(4, 'packages')[i]) + diff --git a/awx/fact/tests/models/fact/fact_simple.py b/awx/fact/tests/models/fact/fact_simple.py index 65b1d960ed..acdfc07d37 100644 --- a/awx/fact/tests/models/fact/fact_simple.py +++ b/awx/fact/tests/models/fact/fact_simple.py @@ -3,66 +3,30 @@ # Python from __future__ import absolute_import -from datetime import datetime +from django.utils.timezone import now from copy import deepcopy +from dateutil.relativedelta import relativedelta # Django # AWX from awx.fact.models.fact import * # noqa -from .base import BaseFactTest +from awx.fact.tests.base import BaseFactTest, FactScanBuilder, TEST_FACT_PACKAGES __all__ = ['FactHostTest', 'FactTest', 'FactGetHostVersionTest', 'FactGetHostTimelineTest'] -TEST_FACT_DATA = { - 'hostname': 'hostname1', - 'add_fact_data': { - 'timestamp': datetime.now(), - 'host': None, - 'module': 'packages', - 'fact': { - "accountsservice": [ - { - "architecture": "amd64", - "name": "accountsservice", - "source": "apt", - "version": "0.6.35-0ubuntu7.1" - } - ], - "acpid": [ - { - "architecture": "amd64", - "name": "acpid", - "source": "apt", - "version": "1:2.0.21-1ubuntu2" - } - ], - "adduser": [ - { - "architecture": "all", - "name": "adduser", - "source": "apt", - "version": "3.113+nmu3ubuntu3" - } - ], - }, - } -} -# Strip off microseconds because mongo has less precision -BaseFactTest.normalize_timestamp(TEST_FACT_DATA) - class FactHostTest(BaseFactTest): def test_create_host(self): - host = FactHost(hostname=TEST_FACT_DATA['hostname']) + host = FactHost(hostname='hosty') host.save() - host = FactHost.objects.get(hostname=TEST_FACT_DATA['hostname']) + host = FactHost.objects.get(hostname='hosty') self.assertIsNotNone(host, "Host added but not found") - self.assertEqual(TEST_FACT_DATA['hostname'], host.hostname, "Gotten record hostname does not match expected hostname") + self.assertEqual('hosty', host.hostname, "Gotten record hostname does not match expected hostname") # Ensure an error is raised for .get() that doesn't match a record. def test_get_host_id_no_result(self): - host = FactHost(hostname=TEST_FACT_DATA['hostname']) + host = FactHost(hostname='hosty') host.save() self.assertRaises(FactHost.DoesNotExist, FactHost.objects.get, hostname='doesnotexist') @@ -70,70 +34,64 @@ class FactHostTest(BaseFactTest): class FactTest(BaseFactTest): def setUp(self): super(FactTest, self).setUp() - self.create_host_document(TEST_FACT_DATA) def test_add_fact(self): - (f_obj, v_obj) = Fact.add_fact(**TEST_FACT_DATA['add_fact_data']) + timestamp = now().replace(microsecond=0) + host = FactHost(hostname="hosty").save() + (f_obj, v_obj) = Fact.add_fact(host=host, timestamp=timestamp, module='packages', fact=TEST_FACT_PACKAGES) f = Fact.objects.get(id=f_obj.id) v = FactVersion.objects.get(id=v_obj.id) self.assertEqual(f.id, f_obj.id) - self.assertEqual(f.module, TEST_FACT_DATA['add_fact_data']['module']) - self.assertEqual(f.fact, TEST_FACT_DATA['add_fact_data']['fact']) - self.assertEqual(f.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp']) + self.assertEqual(f.module, 'packages') + self.assertEqual(f.fact, TEST_FACT_PACKAGES) + self.assertEqual(f.timestamp, timestamp) # host relationship created - self.assertEqual(f.host.id, TEST_FACT_DATA['add_fact_data']['host'].id) + self.assertEqual(f.host.id, host.id) # version created and related self.assertEqual(v.id, v_obj.id) - self.assertEqual(v.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp']) - self.assertEqual(v.host.id, TEST_FACT_DATA['add_fact_data']['host'].id) + self.assertEqual(v.timestamp, timestamp) + self.assertEqual(v.host.id, host.id) self.assertEqual(v.fact.id, f_obj.id) - self.assertEqual(v.fact.module, TEST_FACT_DATA['add_fact_data']['module']) + self.assertEqual(v.fact.module, 'packages') class FactGetHostVersionTest(BaseFactTest): def setUp(self): super(FactGetHostVersionTest, self).setUp() - self.create_host_document(TEST_FACT_DATA) - - self.t1 = datetime.now().replace(second=1, microsecond=0) - self.t2 = datetime.now().replace(second=2, microsecond=0) - data = deepcopy(TEST_FACT_DATA) - data['add_fact_data']['timestamp'] = self.t1 - (self.f1, self.v1) = Fact.add_fact(**data['add_fact_data']) - data = deepcopy(TEST_FACT_DATA) - data['add_fact_data']['timestamp'] = self.t2 - (self.f2, self.v2) = Fact.add_fact(**data['add_fact_data']) + self.builder = FactScanBuilder() + self.builder.add_fact('packages', TEST_FACT_PACKAGES) + self.builder.build(scan_count=2, host_count=1) def test_get_host_version_exact_timestamp(self): - fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=self.t1, module=TEST_FACT_DATA['add_fact_data']['module']) - self.assertIsNotNone(fact, "Set of Facts not found") - self.assertEqual(self.f1.id, fact.id) - self.assertEqual(self.f1.fact, fact.fact) + fact_known = self.builder.get_scan(0, 'packages')[0] + fact = Fact.get_host_version(hostname=self.builder.get_hostname(0), timestamp=self.builder.get_timestamp(0), module='packages') + self.assertIsNotNone(fact) + self.assertEqual(fact_known, fact) def test_get_host_version_lte_timestamp(self): - t3 = datetime.now().replace(second=3, microsecond=0) - fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module']) - self.assertEqual(self.f1.id, fact.id) - self.assertEqual(self.f1.fact, fact.fact) + timestamp = self.builder.get_timestamp(0) + relativedelta(days=1) + fact_known = self.builder.get_scan(0, 'packages')[0] + fact = Fact.get_host_version(hostname=self.builder.get_hostname(0), timestamp=timestamp, module='packages') + self.assertIsNotNone(fact) + self.assertEqual(fact_known, fact) def test_get_host_version_none(self): - t3 = deepcopy(self.t1).replace(second=0) - fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module']) + timestamp = self.builder.get_timestamp(0) - relativedelta(years=20) + fact = Fact.get_host_version(hostname=self.builder.get_hostname(0), timestamp=timestamp, module='packages') self.assertIsNone(fact) class FactGetHostTimelineTest(BaseFactTest): def setUp(self): super(FactGetHostTimelineTest, self).setUp() - #self.create_host_document(TEST_FACT_DATA) - - self.scans = 20 - self.timestamps = self.create_fact_scans(TEST_FACT_DATA, host_count=1, scan_count=self.scans) + self.builder = FactScanBuilder() + self.builder.add_fact('packages', TEST_FACT_PACKAGES) + self.builder.build(scan_count=20, host_count=1) def test_get_host_timeline_ok(self): - timestamps = Fact.get_host_timeline(hostname=self.hostnames[0].hostname, module=TEST_FACT_DATA['add_fact_data']['module']) + timestamps = Fact.get_host_timeline(hostname=self.builder.get_hostname(0), module='packages') self.assertIsNotNone(timestamps) - self.assertEqual(len(timestamps), len(self.timestamps)) - for i in range(0, self.scans): - self.assertEqual(timestamps[i], self.timestamps[i]) + self.assertEqual(len(timestamps), self.builder.get_scan_count()) + for i in range(0, self.builder.get_scan_count()): + self.assertEqual(timestamps[i], self.builder.get_timestamp(i)) diff --git a/awx/fact/tests/models/fact/fact_transform.py b/awx/fact/tests/models/fact/fact_transform.py index 6661f81179..82c27fae9c 100644 --- a/awx/fact/tests/models/fact/fact_transform.py +++ b/awx/fact/tests/models/fact/fact_transform.py @@ -13,38 +13,45 @@ import pymongo # AWX from awx.fact.models.fact import * # noqa -from .base import BaseFactTest +from awx.fact.tests.base import BaseFactTest __all__ = ['FactTransformTest', 'FactTransformUpdateTest',] -TEST_FACT_DATA = { - 'hostname': 'hostname1', - 'add_fact_data': { - 'timestamp': datetime.now(), - 'host': None, - 'module': 'packages', - 'fact': { - "acpid3.4": [ - { - "version": "1:2.0.21-1ubuntu2", - "deeper.key": "some_value" - } - ], - "adduser.2": [ - { - "source": "apt", - "version": "3.113+nmu3ubuntu3" - } - ], - "what.ever." : { - "shallowish.key": "some_shallow_value" - } - }, +TEST_FACT_PACKAGES_WITH_DOTS = [ + { + "name": "acpid3.4", + "version": "1:2.0.21-1ubuntu2", + "deeper.key": "some_value" + }, + { + "name": "adduser.2", + "source": "apt", + "version": "3.113+nmu3ubuntu3" + }, + { + "what.ever." : { + "shallowish.key": "some_shallow_value" + } } -} -# Strip off microseconds because mongo has less precision -BaseFactTest.normalize_timestamp(TEST_FACT_DATA) +] +TEST_FACT_PACKAGES_WITH_DOLLARS = [ + { + "name": "acpid3$4", + "version": "1:2.0.21-1ubuntu2", + "deeper.key": "some_value" + }, + { + "name": "adduser$2", + "source": "apt", + "version": "3.113+nmu3ubuntu3" + }, + { + "what.ever." : { + "shallowish.key": "some_shallow_value" + } + } +] class FactTransformTest(BaseFactTest): def setUp(self): super(FactTransformTest, self).setUp() @@ -52,16 +59,16 @@ class FactTransformTest(BaseFactTest): self.client = pymongo.MongoClient('localhost', 27017) self.db2 = self.client[settings.MONGO_DB] - self.create_host_document(TEST_FACT_DATA) + self.timestamp = datetime.now().replace(microsecond=0) def setup_create_fact_dot(self): - self.data = TEST_FACT_DATA - self.f = Fact(**TEST_FACT_DATA['add_fact_data']) + self.host = FactHost(hostname='hosty').save() + self.f = Fact(timestamp=self.timestamp, module='packages', fact=TEST_FACT_PACKAGES_WITH_DOTS, host=self.host) self.f.save() def setup_create_fact_dollar(self): - self.data = TEST_FACT_DATA - self.f = Fact(**TEST_FACT_DATA['add_fact_data']) + self.host = FactHost(hostname='hosty').save() + self.f = Fact(timestamp=self.timestamp, module='packages', fact=TEST_FACT_PACKAGES_WITH_DOLLARS, host=self.host) self.f.save() def test_fact_with_dot_serialized(self): @@ -73,17 +80,18 @@ class FactTransformTest(BaseFactTest): # Bypass mongoengine and pymongo transform to get record f_dict = self.db2['fact'].find_one(q) - self.assertIn('acpid3\uff0E4', f_dict['fact']) + self.assertIn('what\uff0Eever\uff0E', f_dict['fact'][2]) def test_fact_with_dot_serialized_pymongo(self): #self.setup_create_fact_dot() + host = FactHost(hostname='hosty').save() f = self.db['fact'].insert({ - 'hostname': TEST_FACT_DATA['hostname'], - 'fact': TEST_FACT_DATA['add_fact_data']['fact'], - 'timestamp': TEST_FACT_DATA['add_fact_data']['timestamp'], - 'host': TEST_FACT_DATA['add_fact_data']['host'].id, - 'module': TEST_FACT_DATA['add_fact_data']['module'] + 'hostname': 'hosty', + 'fact': TEST_FACT_PACKAGES_WITH_DOTS, + 'timestamp': self.timestamp, + 'host': host.id, + 'module': 'packages', }) q = { @@ -91,7 +99,7 @@ class FactTransformTest(BaseFactTest): } # Bypass mongoengine and pymongo transform to get record f_dict = self.db2['fact'].find_one(q) - self.assertIn('acpid3\uff0E4', f_dict['fact']) + self.assertIn('what\uff0Eever\uff0E', f_dict['fact'][2]) def test_fact_with_dot_deserialized_pymongo(self): self.setup_create_fact_dot() @@ -100,13 +108,13 @@ class FactTransformTest(BaseFactTest): '_id': self.f.id } f_dict = self.db['fact'].find_one(q) - self.assertIn('acpid3.4', f_dict['fact']) + self.assertIn('what.ever.', f_dict['fact'][2]) def test_fact_with_dot_deserialized(self): self.setup_create_fact_dot() f = Fact.objects.get(id=self.f.id) - self.assertIn('acpid3.4', f.fact) + self.assertIn('what.ever.', f.fact[2]) class FactTransformUpdateTest(BaseFactTest): pass diff --git a/awx/fact/tests/models/fact/fact_transform_pymongo.py b/awx/fact/tests/models/fact/fact_transform_pymongo.py index 7cf81e4650..ac7c329980 100644 --- a/awx/fact/tests/models/fact/fact_transform_pymongo.py +++ b/awx/fact/tests/models/fact/fact_transform_pymongo.py @@ -13,7 +13,7 @@ import pymongo # AWX from awx.fact.models.fact import * # noqa -from .base import BaseFactTest +from awx.fact.tests.base import BaseFactTest __all__ = ['FactSerializePymongoTest', 'FactDeserializePymongoTest',] diff --git a/awx/main/tests/__init__.py b/awx/main/tests/__init__.py index dda898b544..3b5e1fcf12 100644 --- a/awx/main/tests/__init__.py +++ b/awx/main/tests/__init__.py @@ -16,3 +16,4 @@ from awx.main.tests.schedules import * # noqa from awx.main.tests.redact import * # noqa from awx.main.tests.views import * # noqa from awx.main.tests.commands import * # noqa +from awx.main.tests.fact import * # noqa diff --git a/awx/main/tests/base.py b/awx/main/tests/base.py index a7f83ace09..8fdef66623 100644 --- a/awx/main/tests/base.py +++ b/awx/main/tests/base.py @@ -25,9 +25,6 @@ from django.contrib.auth.models import User from django.test.client import Client from django.test.utils import override_settings -# MongoEngine -from mongoengine.connection import get_db, ConnectionError - # AWX from awx.main.models import * # noqa from awx.main.backend import LDAPSettings @@ -43,15 +40,6 @@ TEST_PLAYBOOK = '''- hosts: mygroup command: test 1 = 1 ''' -class MongoDBRequired(django.test.TestCase): - def setUp(self): - # Drop mongo database - try: - self.db = get_db() - self.db.connection.drop_database(settings.MONGO_DB) - except ConnectionError: - self.skipTest('MongoDB connection failed') - class QueueTestMixin(object): def start_queue(self): self.start_redis() diff --git a/awx/main/tests/commands/cleanup_facts.py b/awx/main/tests/commands/cleanup_facts.py index 8d02310a2d..cdddce9f8f 100644 --- a/awx/main/tests/commands/cleanup_facts.py +++ b/awx/main/tests/commands/cleanup_facts.py @@ -10,7 +10,8 @@ import mock from django.core.management.base import CommandError # AWX -from awx.main.tests.base import BaseTest, MongoDBRequired +from awx.main.tests.base import BaseTest +from awx.fact.tests.base import MongoDBRequired from awx.main.tests.commands.base import BaseCommandMixin from awx.main.management.commands.cleanup_facts import Command, CleanupFacts from awx.fact.models.fact import * # noqa diff --git a/awx/main/tests/commands/run_fact_cache_receiver.py b/awx/main/tests/commands/run_fact_cache_receiver.py index 3933813ff4..b8a2474111 100644 --- a/awx/main/tests/commands/run_fact_cache_receiver.py +++ b/awx/main/tests/commands/run_fact_cache_receiver.py @@ -10,7 +10,8 @@ from copy import deepcopy from mock import MagicMock # AWX -from awx.main.tests.base import BaseTest, MongoDBRequired +from awx.main.tests.base import BaseTest +from awx.fact.tests.base import MongoDBRequired from awx.main.tests.commands.base import BaseCommandMixin from awx.main.management.commands.run_fact_cache_receiver import FactCacheReceiver from awx.fact.models.fact import * # noqa diff --git a/awx/main/tests/fact/__init__.py b/awx/main/tests/fact/__init__.py new file mode 100644 index 0000000000..234499d6e9 --- /dev/null +++ b/awx/main/tests/fact/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +from __future__ import absolute_import + +from .fact_api import * # noqa diff --git a/awx/main/tests/fact/fact_api.py b/awx/main/tests/fact/fact_api.py new file mode 100644 index 0000000000..429e382fe3 --- /dev/null +++ b/awx/main/tests/fact/fact_api.py @@ -0,0 +1,184 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved + +# Python + +# Django +import django +from django.core.urlresolvers import reverse +from django.utils.timezone import now + +# AWX +from awx.main.utils import timestamp_apiformat +from awx.main.models import * # noqa +from awx.main.tests.base import BaseLiveServerTest +from awx.fact.models import * # noqa +from awx.fact.tests.base import BaseFactTestMixin, FactScanBuilder, TEST_FACT_ANSIBLE, TEST_FACT_PACKAGES, TEST_FACT_SERVICES +from awx.main.utils import build_url + +__all__ = ['FactVersionApiTest', 'FactViewApiTest'] + +class FactApiBaseTest(BaseLiveServerTest, BaseFactTestMixin): + def setUp(self): + super(FactApiBaseTest, self).setUp() + self.setup_instances() + self.setup_users() + self.organization = self.make_organization(self.super_django_user) + self.organization.admins.add(self.normal_django_user) + self.inventory = self.organization.inventories.create(name='test-inventory', description='description for test-inventory') + self.host = self.inventory.hosts.create(name='host.example.com') + self.host2 = self.inventory.hosts.create(name='host2.example.com') + self.host3 = self.inventory.hosts.create(name='host3.example.com') + + def setup_facts(self, scan_count): + self.builder = FactScanBuilder() + self.builder.add_fact('ansible', TEST_FACT_ANSIBLE) + self.builder.add_fact('packages', TEST_FACT_PACKAGES) + self.builder.add_fact('services', TEST_FACT_SERVICES) + self.builder.add_hostname('host.example.com') + self.builder.add_hostname('host2.example.com') + self.builder.add_hostname('host3.example.com') + self.builder.build(scan_count=scan_count, host_count=3) + + self.fact_host = FactHost.objects.get(hostname=self.host.name) + +class FactVersionApiTest(FactApiBaseTest): + def check_equal(self, fact_versions, results): + def find(element, set1): + for e in set1: + if all([ e.get(field) == element.get(field) for field in element.keys()]): + return e + return None + + self.assertEqual(len(results), len(fact_versions)) + for v in fact_versions: + v_dict = { + 'timestamp': timestamp_apiformat(v.timestamp), + 'module': v.module + } + e = find(v_dict, results) + self.assertIsNotNone(e, "%s not found in %s" % (v_dict, results)) + + def get_list(self, fact_versions, params=None): + url = build_url('api:host_fact_versions_list', args=(self.host.pk,), get=params) + with self.current_user(self.super_django_user): + response = self.get(url, expect=200) + + self.check_equal(fact_versions, response['results']) + return response + + def test_permission_list(self): + url = reverse('api:host_fact_versions_list', args=(self.host.pk,)) + with self.current_user('admin'): + self.get(url, expect=200) + with self.current_user('normal'): + self.get(url, expect=200) + with self.current_user('other'): + self.get(url, expect=403) + with self.current_user('nobody'): + self.get(url, expect=403) + with self.current_user(None): + self.get(url, expect=401) + + def test_list_empty(self): + url = reverse('api:host_fact_versions_list', args=(self.host.pk,)) + with self.current_user(self.super_django_user): + response = self.get(url, expect=200) + self.assertIn('results', response) + self.assertIsInstance(response['results'], list) + self.assertEqual(len(response['results']), 0) + + def test_list_related_fact_view(self): + self.setup_facts(2) + url = reverse('api:host_fact_versions_list', args=(self.host.pk,)) + with self.current_user(self.super_django_user): + response = self.get(url, expect=200) + for entry in response['results']: + self.assertIn('fact_view', entry['related']) + r = self.get(entry['related']['fact_view'], expect=200) + print(r) + + def test_list(self): + self.setup_facts(2) + self.get_list(FactVersion.objects.filter(host=self.fact_host)) + + def test_list_module(self): + self.setup_facts(10) + self.get_list(FactVersion.objects.filter(host=self.fact_host, module='packages'), dict(module='packages')) + + def test_list_time_from(self): + self.setup_facts(10) + + params = { + 'from': timestamp_apiformat(self.builder.get_timestamp(1)), + } + # 'to': timestamp_apiformat(self.builder.get_timestamp(3)) + fact_versions = FactVersion.objects.filter(host=self.fact_host, timestamp__gt=params['from']) + self.get_list(fact_versions, params) + + def test_list_time_to(self): + self.setup_facts(10) + + params = { + 'to': timestamp_apiformat(self.builder.get_timestamp(3)) + } + fact_versions = FactVersion.objects.filter(host=self.fact_host, timestamp__lte=params['to']) + self.get_list(fact_versions, params) + + def test_list_time_from_to(self): + self.setup_facts(10) + + params = { + 'from': timestamp_apiformat(self.builder.get_timestamp(1)), + 'to': timestamp_apiformat(self.builder.get_timestamp(3)) + } + fact_versions = FactVersion.objects.filter(host=self.fact_host, timestamp__gt=params['from'], timestamp__lte=params['to']) + self.get_list(fact_versions, params) + + +class FactViewApiTest(FactApiBaseTest): + def check_equal(self, fact_obj, results): + fact_dict = { + 'timestamp': timestamp_apiformat(fact_obj.timestamp), + 'module': fact_obj.module, + 'host': { + 'hostname': fact_obj.host.hostname, + 'id': str(fact_obj.host.id) + }, + 'fact': fact_obj.fact + } + self.assertEqual(fact_dict, results) + + def test_permission_view(self): + url = reverse('api:host_fact_compare_view', args=(self.host.pk,)) + with self.current_user('admin'): + self.get(url, expect=200) + with self.current_user('normal'): + self.get(url, expect=200) + with self.current_user('other'): + self.get(url, expect=403) + with self.current_user('nobody'): + self.get(url, expect=403) + with self.current_user(None): + self.get(url, expect=401) + + def get_fact(self, fact_obj, params=None): + url = build_url('api:host_fact_compare_view', args=(self.host.pk,), get=params) + with self.current_user(self.super_django_user): + response = self.get(url, expect=200) + + self.check_equal(fact_obj, response) + + def test_view(self): + self.setup_facts(2) + self.get_fact(Fact.objects.filter(host=self.fact_host, module='ansible').order_by('-timestamp')[0]) + + def test_view_module_filter(self): + self.setup_facts(2) + self.get_fact(Fact.objects.filter(host=self.fact_host, module='services').order_by('-timestamp')[0], dict(module='services')) + + def test_view_time_filter(self): + self.setup_facts(6) + ts = self.builder.get_timestamp(3) + self.get_fact(Fact.objects.filter(host=self.fact_host, module='ansible', timestamp__lte=ts).order_by('-timestamp')[0], + dict(datetime=ts)) diff --git a/awx/main/utils.py b/awx/main/utils.py index be8c44e890..525e0fcf7f 100644 --- a/awx/main/utils.py +++ b/awx/main/utils.py @@ -15,11 +15,12 @@ import urlparse import threading import contextlib import tempfile +import urllib # Django REST Framework from rest_framework.exceptions import ParseError, PermissionDenied from django.utils.encoding import smart_str - +from django.core.urlresolvers import reverse # PyCrypto from Crypto.Cipher import AES @@ -487,3 +488,16 @@ def get_pk_from_dict(_dict, key): return int(_dict[key]) except (TypeError, KeyError, ValueError): return None + +def build_url(*args, **kwargs): + get = kwargs.pop('get', {}) + url = reverse(*args, **kwargs) + if get: + url += '?' + urllib.urlencode(get) + return url + +def timestamp_apiformat(timestamp): + timestamp = timestamp.isoformat() + if timestamp.endswith('+00:00'): + timestamp = timestamp[:-6] + 'Z' + return timestamp